apxtrib/api/models/Toolsbox.js

647 lines
22 KiB
JavaScript
Raw Normal View History

2023-04-13 05:46:35 +00:00
/* eslint-disable no-useless-escape */
const fs = require("fs");
const path = require("path");
const bcrypt = require("bcrypt");
const moment = require("moment");
const utils = {};
console.log(
"Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apxtrib project (see /package.json to get email).\n We'll add to the roadmap to add it."
);
/**
* EMAIL
*/
/* const validateEmail = email => {
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
return regExp.test(email);
};
const validatePassword = pwd => {
const regExp = new RegExp(
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
);
return regExp.test(pwd);
};
const filterInvalidInArray = (array, validate) =>
array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid
/**
* POSTAL CODE
*/
/*
const validatePostalCode = postalCode =>
/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode);
/**
* PHONE
*/
/* const validatePhoneNumber = phoneNumber =>
/((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber);
const correctPhoneNumber = phone =>
phone[0] === '0' ? '+33' + phone.substr(1) : phone;
const Checkjson = (appProfil, referential, data) => {
// @TODO get a referentiel per object then check data validity and allowed access
// need to add referentiel manager
const invalidefor = [];
let updateDatabase = false;
Object.keys(data).forEach(field => {
switch (field) {
case 'token':
updateDatabase = true;
break;
case 'email':
if (!validateEmail(data.email)) {
invalidefor.push('ERREMAIL:' + field);
} else {
updateDatabase = true;
}
break;
case 'password':
if (!validatePassword(data.password)) {
invalidefor.push('ERRPWD:' + field);
} else {
data.password = bcrypt.hash(data.password, config.saltRounds);
updateDatabase = true;
}
break;
}
});
return { invalidefor, data, updateDatabase };
};
*/
//Permet d'attendre en milliseconde
// s'utilise avec async ()=>{
// await sleep(2000)
//}
utils.sleep = (ms) => {
return new Promise((resolve) => setTimeout(resolve, ms));
};
utils.normalize = {};
utils.normalize.telephonefr = (phone) => {
phone = phone.trim().replace(/[- .]/g, "");
if (
Checkjson.schema.properties.format.telephoenfr(phone) &&
phone.length == 10 &&
phone[0] == "0"
) {
phone = "+33 " + phone.substring(1);
}
return phone;
};
utils.normalize.zfill10 = (num) => {
let s = num + "";
while (s.length < 10) s = "0" + s;
return s;
};
utils.generemdp = (nbpos, fromchar) => {
if (!fromchar) {
const fromchar = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
}
//const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
let mdp = "";
for (var i = 0; i < nbpos; i++) {
var pos = Math.floor(Math.random() * fromchar.length);
mdp += fromchar.substring(pos, pos + 1);
}
return mdp;
};
utils.generecompteur = (filecpt, typeincrement) => {
let file = `${filecpt}/${typeincrement}.json`;
let prefix = "";
if ((typeincrement = "ANNEESEMAINE")) {
file = `${filecpt}/${typeincrement}${moment().format(
"YYYY"
)}${moment().format("WW")}.json`;
prefix = `${moment().format("YYYY")}${moment().format("WW")}`;
}
let num = 1;
try {
num = parseInt(fs.readFileSync(file, "utf8")) + 1;
} catch (err) {
console.log("Nouveau compteur incrementale ", file);
}
fs.writeFileSync(file, num, "utf8");
return prefix + num;
};
/**
* CSV
*/
utils.json2csv = (jsondata, options, callback) => {
// uniquement json = [{niv1:val,niv1:[liste of val]}]
// console.log('_________________________');
// console.log(jsondata)
// console.log('_________________________');
if (jsondata.length == 0) {
return callback("Empty json", null);
}
if (!options.retln) options.retln = "\n";
if (!options.sep) options.sep = ";";
if (!options.arraysplitsep) options.arraysplitsep = ",";
if (!options.replacespecialcarJson2Csv) {
options.replacespecialcarJson2Csv = [];
} else {
if (typeof options.replacespecialcarJson2Csv == "string") {
//permet de passer des regex en string
options.replacespecialcarJson2Csv = eval(
options.replacespecialcarJson2Csv
);
}
}
let etat = "";
let csv = "";
let entete = "";
let prem = true;
for (const j in jsondata) {
// console.log(jsondata[j])
for (const c in options.champs) {
if (prem) {
entete += options.champs[c] + options.sep;
}
if (jsondata[j][options.champs[c]]) {
if (options.array.indexOf(options.champs[c]) > -1) {
csv +=
jsondata[j][options.champs[c]].join(options.arraysplitsep) +
options.sep;
} else {
let currentValue = "";
if (jsondata[j][options.champs[c]])
currentValue += jsondata[j][options.champs[c]];
options.replacespecialcarJson2Csv.forEach((re) => {
//console.log(currentValue)
currentValue = currentValue.replace(re[1], re[0]);
});
csv += currentValue + options.sep;
}
} else {
csv += options.sep;
}
}
csv = csv.substring(0, csv.length - 1) + options.retln;
if (prem) {
prem = false;
entete = entete.substring(0, entete.length - 1) + options.retln;
// console.log(entete)
}
}
// return entete + csv;
if (etat == "") {
return callback(null, entete + csv);
} else {
return callback(etat, null);
}
};
/**
* Get headers from first line of CSV
* @param {array} lines array of string which contains each csv lines
* @return {array} string array of headers
*/
utils.getHeaders = (lines, sep) =>
lines[0].split(sep).map((i) => i.replace(/"/g, ""));
/**
* [csv2json description]
* @param {object} csv object of csv file that has been read
* @param {object} options object containing csv options, headers, ...
{retln:'code de retour de ligne \n ou \n\r',
sep:'code to split cells',
champs:[ch1,ch2,...] catch only those field,
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
* @param {Function} callback callback function
* @return {callback} - return an error if error, else return json
it convert a csv file into a json = [{field:value}]
Usage example:
fiche.csv2article = (err, fiche) => {
if (!err) {
console.log(fiche)
}
}
utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
retln: "\n",
sep: ";",
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
}, fiche.csv2article)
*/
utils.replacecarbtweendblquote = (csv, car, carremplacant) => {
/*
return csv text with any car betwenn 2 " by CARSEPARATOR
*/
let newcsv = "";
let txtencours = "";
let flagouvert = false;
const sepreg = new RegExp(`${car}`, "gmi");
for (let j = 0; j < csv.length; j++) {
//if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) {
if (csv[j] == '"') {
if (flagouvert) {
// on cherche à ferme une chaine de texte
if (csv[j + 1] == '"') {
//on a "" consecutif qu'on remplace par "" et on fait j+1
txtencours += '""';
j++;
} else {
// on a bien une fermeture
flagouvert = false;
newcsv += txtencours.replace(sepreg, carremplacant);
txtencours = '"';
}
} else {
// on ouvre une chaine
flagouvert = true;
//on met le contenu précédent ds newcsv
newcsv += txtencours;
txtencours = '"';
}
//} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) {
} else if (csv[j] !== "\n") {
txtencours += csv[j];
// } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) {
} else if (csv[j] == "\n") {
if (!flagouvert) txtencours += "\n";
}
}
return newcsv + txtencours;
};
utils.analysestring = (string) => {
let buftxt = "";
let bufcode = "";
let i = 0;
let avecRL = false;
for (let p = 0; p < string.length; p++) {
if (string[p].charCodeAt() == 10) {
buftxt += "[RL]";
avecRL = true;
} else {
buftxt += string[p];
}
bufcode += "-" + string[p].charCodeAt();
if (i == 20) {
if (avecRL) {
console.log(`${buftxt} - ${bufcode}`);
} else {
console.log(`${buftxt} ---- ${bufcode}`);
}
i = 0;
buftxt = "";
bufcode = "";
avecRL = false;
}
i++;
}
};
const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","<p>&nbsp;</p>
<table>
<tbody>
<tr>
<td>
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
</td>
</tr>
</tbody>
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0,
32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","<p>&nbsp;</p>
<table>
<tbody>
<tr>
<td>
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
</td>
</tr>
</tbody>
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0,
32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0,
32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`;
//utils.analysestring(txtstring)
//console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR')
// .split("\n")[0].split(","))
utils.csv2json = (csv, options, callback) => {
// EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT
// l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché
console.log("\n--------------- CSV2JSON ---------------\n");
// Default CSV options
if (!options.retln) options.retln = "\n";
if (csv.indexOf("\n\r") > -1) options.retln = "\n\r";
if (!options.sep) options.sep = ";";
//gestion d un separateur dans une chaine de texte
//const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm');
//csv = csv.replace(regseptext, "CARACSEPAR");
// csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE")
csv = utils.replacecarbtweendblquote(csv, options.sep, "CARSEPARATOR");
if (!options.replacespecialcarCsv2Json) {
options.replacespecialcarCsv2Json = [];
} else {
if (typeof options.replacespecialcarCsv2Json == "string") {
//permet de passer des regex en string
options.replacespecialcarCsv2Json = eval(
options.replacespecialcarCsv2Json
);
}
}
const result = [];
const lines = csv.split(options.retln);
const headers = utils.getHeaders(lines, options.sep);
let unknownHeaders = "";
//console.log('headers', headers)
//console.log('options.champs', options.champs)
headers.forEach((header) => {
// Si un header n'est pas présent dans la liste des champs prédéfinis
// on l'ajoute aux champs inconnus
if (options.champs.indexOf(header) === -1) {
unknownHeaders += `${header}, `;
}
});
if (unknownHeaders !== "") {
const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`;
return callback(errorMsg, null);
}
lines.forEach((line, index) => {
// Skip headers line or empty lines
if (index === 0 || line.replace(/\s/g, "").length === 0) {
return;
}
// pour debuguer on met origincsv pour voir la ligne d'origine
const currentLineData = { origincsv: line, linenumber: index };
const currentLine = line.split(options.sep); // Current string in the line
for (let j = 0; j < headers.length; j++) {
// Si la ligne n'est pas vide
if (currentLine[j]) {
// On clean le champs
// ajout eventuel de modification de caracter reservé ; dans les libelléetc...
let currentValue = currentLine[j].trim();
//on transforme le caractere separateur modifié entre double quote
currentValue = currentValue.replace("CARSEPARATOR", options.sep);
options.replacespecialcarCsv2Json.forEach((re) => {
currentValue = currentValue.replace(re[0], re[1]);
});
// Si le header est un email
if (headers[j].includes("EMAIL")) {
// Supprimer tous les espaces
currentValue = currentLine[j].replace(/\s/g, "");
}
// on check si le chamos doit être numerique
if (options.numericfield.includes(headers[j])) {
currentValue = currentLine[j].replace(/\,/g, ".");
try {
const test = parseFloat(currentValue);
} catch (er) {
return callback(
`${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`,
null
);
}
}
if (currentValue) {
// Si le header actuel est de type array
// Cela signifie que le header apparaît plusieurs fois dans le CSV
// et que les valeurs correspondantes à ce header
// doivent être mis dans un array
if (options.array && options.array.indexOf(headers[j]) > -1) {
// Si le tableau pour ce header n'existe pas on le crée
if (!currentLineData[headers[j]]) {
currentLineData[headers[j]] = [];
}
if (options.arraysplitsep) {
currentValue.split(options.arraysplitsep).forEach((v) => {
currentLineData[headers[j]].push(v);
});
} else {
currentLineData[headers[j]].push(currentValue);
}
} else {
// Si un header est déjà présent pour la ligne
// alors que il n'est pas spécifié comme étant un array
// on retourne une erreur
if (currentLineData[headers[j]]) {
const errorMsg = `Le champ ${headers[j]} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`;
return callback(errorMsg, null);
}
currentLineData[headers[j]] = currentValue;
}
}
}
}
result.push(currentLineData);
});
return callback(null, result);
};
/**
* [csvparam2json description]
* @param {object} csv object of csv file that has been read
* @param {object} options object containing csv options, headers, ...
{retln:'code de retour de ligne \n ou \n\r',
sep:'code to split cells',
champs:[ch1,ch2,...] catch only those field,
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
* @param {Function} callback callback function
* @return {callback} - return an error if error, else return json
it converts a csv with 3 column col1;col2;col3 in a json in a tree
if in col1 we have __ => then it splits a leaf
col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored
return data = {xxxx:{yyyy:value}}
col1 = xxxx; col2 = value; col3 = comment ignored
return data = {xxxx:value}
Usage example:
fiche.csvparam2article = (err, fiche) => {
if (!err) {
console.log(fiche)
}
}
utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
retln: "\n",
sep: ";",
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
}, fiche.csv2article)
*/
utils.csvparam2json = (csv, options, callback) => {
console.log("\n--------------- CSVPARAM2JSON ---------------\n");
let etat = "";
const param = {};
if (!options.retln) {
options.retln = "\n";
}
if (csv.indexOf("\n\r") > -1) {
options.retln = "\n\r";
}
if (!options.sep) {
options.sep = ";";
}
if (!options.seplevel) {
options.seplevel = "__";
}
if (!options.replacespecialcarCsv2Json) {
options.replacespecialcarCsv2Json = [];
} else {
if (typeof options.replacespecialcarCsv2Json == "string") {
//permet de passer des regex en string
options.replacespecialcarCsv2Json = eval(
options.replacespecialcarCsv2Json
);
}
}
const lines = csv.split(options.retln);
for (let i = 0; i < lines.length; i++) {
const infol = lines[i].split(options.sep);
//console.log(infol)
if (infol[0].length > 4 && infol.length < 2) {
// si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb
etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`;
continue;
}
// On ajoute ici la gestion de tous les caracteres spéciaux
// reservées pour le csv ; ' etc..'
if (infol[1] && infol[1] + "" == infol[1]) {
options.replacespecialcarCsv2Json.forEach((re) => {
//console.log("gggggggggggggggggggg", infol[1])
infol[1] = infol[1].replace(re[0], re[1]);
});
// console.log(infol[1])
infol[1] = infol[1].replace(/'|/g, '"');
//console.log(infol[1])
if (infol[1].toLowerCase() === "true") {
infol[1] = true;
} else if (infol[1].toLowerCase() === "false") {
infol[1] = false;
}
}
console.log(infol[1]);
//supprime des lignes vides
if (infol[0] == "") continue;
if (infol[0].indexOf(options.seplevel) == -1) {
param[infol[0]] = infol[1];
continue;
} else {
const arbre = infol[0].split(options.seplevel);
switch (arbre.length) {
case 1:
param[arbre[0]] = infol[1];
break;
case 2:
if (arbre[1] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
param[arbre[0]][arbre[1]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = [];
//console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"'))
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]].push(result);
}
break;
case 3:
if (arbre[2] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
param[arbre[0]][arbre[1]][arbre[2]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = [];
//eval("result = \"test\"");
//console.log(result);
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]][arbre[1]].push(result);
}
break;
case 4:
if (arbre[3] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
if (!param[arbre[0]][arbre[1]][arbre[2]])
param[arbre[0]][arbre[1]][arbre[2]] = {};
param[arbre[0]][arbre[1]][arbre[2]][arbre[3]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
if (!param[arbre[0]][arbre[1]][arbre[2]])
param[arbre[0]][arbre[1]][arbre[2]] = [];
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]][arbre[1]][arbre[2]].push(result);
break;
}
default:
break;
}
}
}
// JSON.parse(JSON.stringify(param))
console.log(
"kkkkkkkkkkkkkkkkkk",
param["catalogue"]["filtrecatalog"]["searchengine"]
);
if (etat == "") {
return callback(null, JSON.parse(JSON.stringify(param)));
} else {
return callback(etat, null);
}
};
utils.levenshtein = (a, b) => {
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
let tmp, i, j, prev, val, row;
// swap to save some memory O(min(a,b)) instead of O(a)
if (a.length > b.length) {
tmp = a;
a = b;
b = tmp;
}
row = Array(a.length + 1);
// init the row
for (i = 0; i <= a.length; i++) {
row[i] = i;
}
// fill in the rest
for (i = 1; i <= b.length; i++) {
prev = i;
for (j = 1; j <= a.length; j++) {
if (b[i - 1] === a[j - 1]) {
val = row[j - 1]; // match
} else {
val = Math.min(
row[j - 1] + 1, // substitution
Math.min(
prev + 1, // insertion
row[j] + 1
)
); // deletion
}
row[j - 1] = prev;
prev = val;
}
row[a.length] = prev;
}
return row[a.length];
};
utils.testinarray = (array, arrayreferent) => {
// au moins un element de array existe dans arryreferent
let exist = false;
if (arrayreferent) {
//console.log('arrrrrrrrrrrrrrr', arrayreferent)
array.forEach((e) => {
//console.log(e)
if (arrayreferent.includes(e)) exist = true;
});
}
return exist;
};
/*
DIRECTORY
*/
const isDirectory = (source) => fs.lstatSync(source).isDirectory();
const getDirectories = (source) =>
fs
.readdirSync(source)
.map((name) => path.join(source, name))
.filter(isDirectory);
module.exports = utils;