apxtrib/api/models/Toolsbox.js
2023-04-27 06:17:20 +02:00

647 lines
22 KiB
JavaScript
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/* eslint-disable no-useless-escape */
const fs = require("fs");
const path = require("path");
const bcrypt = require("bcrypt");
const moment = require("moment");
const utils = {};
console.log(
"Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apxtrib project (see /package.json to get email).\n We'll add to the roadmap to add it."
);
/**
* EMAIL
*/
/* const validateEmail = email => {
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
return regExp.test(email);
};
const validatePassword = pwd => {
const regExp = new RegExp(
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
);
return regExp.test(pwd);
};
const filterInvalidInArray = (array, validate) =>
array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid
/**
* POSTAL CODE
*/
/*
const validatePostalCode = postalCode =>
/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode);
/**
* PHONE
*/
/* const validatePhoneNumber = phoneNumber =>
/((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber);
const correctPhoneNumber = phone =>
phone[0] === '0' ? '+33' + phone.substr(1) : phone;
const Checkjson = (appProfil, referential, data) => {
// @TODO get a referentiel per object then check data validity and allowed access
// need to add referentiel manager
const invalidefor = [];
let updateDatabase = false;
Object.keys(data).forEach(field => {
switch (field) {
case 'token':
updateDatabase = true;
break;
case 'email':
if (!validateEmail(data.email)) {
invalidefor.push('ERREMAIL:' + field);
} else {
updateDatabase = true;
}
break;
case 'password':
if (!validatePassword(data.password)) {
invalidefor.push('ERRPWD:' + field);
} else {
data.password = bcrypt.hash(data.password, config.saltRounds);
updateDatabase = true;
}
break;
}
});
return { invalidefor, data, updateDatabase };
};
*/
//Permet d'attendre en milliseconde
// s'utilise avec async ()=>{
// await sleep(2000)
//}
utils.sleep = (ms) => {
return new Promise((resolve) => setTimeout(resolve, ms));
};
utils.normalize = {};
utils.normalize.telephonefr = (phone) => {
phone = phone.trim().replace(/[- .]/g, "");
if (
Checkjson.schema.properties.format.telephoenfr(phone) &&
phone.length == 10 &&
phone[0] == "0"
) {
phone = "+33 " + phone.substring(1);
}
return phone;
};
utils.normalize.zfill10 = (num) => {
let s = num + "";
while (s.length < 10) s = "0" + s;
return s;
};
utils.generemdp = (nbpos, fromchar) => {
if (!fromchar) {
const fromchar = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
}
//const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
let mdp = "";
for (var i = 0; i < nbpos; i++) {
var pos = Math.floor(Math.random() * fromchar.length);
mdp += fromchar.substring(pos, pos + 1);
}
return mdp;
};
utils.generecompteur = (filecpt, typeincrement) => {
let file = `${filecpt}/${typeincrement}.json`;
let prefix = "";
if ((typeincrement = "ANNEESEMAINE")) {
file = `${filecpt}/${typeincrement}${moment().format(
"YYYY"
)}${moment().format("WW")}.json`;
prefix = `${moment().format("YYYY")}${moment().format("WW")}`;
}
let num = 1;
try {
num = parseInt(fs.readFileSync(file, "utf8")) + 1;
} catch (err) {
console.log("Nouveau compteur incrementale ", file);
}
fs.writeFileSync(file, num, "utf8");
return prefix + num;
};
/**
* CSV
*/
utils.json2csv = (jsondata, options, callback) => {
// uniquement json = [{niv1:val,niv1:[liste of val]}]
// console.log('_________________________');
// console.log(jsondata)
// console.log('_________________________');
if (jsondata.length == 0) {
return callback("Empty json", null);
}
if (!options.retln) options.retln = "\n";
if (!options.sep) options.sep = ";";
if (!options.arraysplitsep) options.arraysplitsep = ",";
if (!options.replacespecialcarJson2Csv) {
options.replacespecialcarJson2Csv = [];
} else {
if (typeof options.replacespecialcarJson2Csv == "string") {
//permet de passer des regex en string
options.replacespecialcarJson2Csv = eval(
options.replacespecialcarJson2Csv
);
}
}
let etat = "";
let csv = "";
let entete = "";
let prem = true;
for (const j in jsondata) {
// console.log(jsondata[j])
for (const c in options.champs) {
if (prem) {
entete += options.champs[c] + options.sep;
}
if (jsondata[j][options.champs[c]]) {
if (options.array.indexOf(options.champs[c]) > -1) {
csv +=
jsondata[j][options.champs[c]].join(options.arraysplitsep) +
options.sep;
} else {
let currentValue = "";
if (jsondata[j][options.champs[c]])
currentValue += jsondata[j][options.champs[c]];
options.replacespecialcarJson2Csv.forEach((re) => {
//console.log(currentValue)
currentValue = currentValue.replace(re[1], re[0]);
});
csv += currentValue + options.sep;
}
} else {
csv += options.sep;
}
}
csv = csv.substring(0, csv.length - 1) + options.retln;
if (prem) {
prem = false;
entete = entete.substring(0, entete.length - 1) + options.retln;
// console.log(entete)
}
}
// return entete + csv;
if (etat == "") {
return callback(null, entete + csv);
} else {
return callback(etat, null);
}
};
/**
* Get headers from first line of CSV
* @param {array} lines array of string which contains each csv lines
* @return {array} string array of headers
*/
utils.getHeaders = (lines, sep) =>
lines[0].split(sep).map((i) => i.replace(/"/g, ""));
/**
* [csv2json description]
* @param {object} csv object of csv file that has been read
* @param {object} options object containing csv options, headers, ...
{retln:'code de retour de ligne \n ou \n\r',
sep:'code to split cells',
champs:[ch1,ch2,...] catch only those field,
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
* @param {Function} callback callback function
* @return {callback} - return an error if error, else return json
it convert a csv file into a json = [{field:value}]
Usage example:
fiche.csv2article = (err, fiche) => {
if (!err) {
console.log(fiche)
}
}
utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
retln: "\n",
sep: ";",
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
}, fiche.csv2article)
*/
utils.replacecarbtweendblquote = (csv, car, carremplacant) => {
/*
return csv text with any car betwenn 2 " by CARSEPARATOR
*/
let newcsv = "";
let txtencours = "";
let flagouvert = false;
const sepreg = new RegExp(`${car}`, "gmi");
for (let j = 0; j < csv.length; j++) {
//if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) {
if (csv[j] == '"') {
if (flagouvert) {
// on cherche à ferme une chaine de texte
if (csv[j + 1] == '"') {
//on a "" consecutif qu'on remplace par "" et on fait j+1
txtencours += '""';
j++;
} else {
// on a bien une fermeture
flagouvert = false;
newcsv += txtencours.replace(sepreg, carremplacant);
txtencours = '"';
}
} else {
// on ouvre une chaine
flagouvert = true;
//on met le contenu précédent ds newcsv
newcsv += txtencours;
txtencours = '"';
}
//} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) {
} else if (csv[j] !== "\n") {
txtencours += csv[j];
// } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) {
} else if (csv[j] == "\n") {
if (!flagouvert) txtencours += "\n";
}
}
return newcsv + txtencours;
};
utils.analysestring = (string) => {
let buftxt = "";
let bufcode = "";
let i = 0;
let avecRL = false;
for (let p = 0; p < string.length; p++) {
if (string[p].charCodeAt() == 10) {
buftxt += "[RL]";
avecRL = true;
} else {
buftxt += string[p];
}
bufcode += "-" + string[p].charCodeAt();
if (i == 20) {
if (avecRL) {
console.log(`${buftxt} - ${bufcode}`);
} else {
console.log(`${buftxt} ---- ${bufcode}`);
}
i = 0;
buftxt = "";
bufcode = "";
avecRL = false;
}
i++;
}
};
const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","<p>&nbsp;</p>
<table>
<tbody>
<tr>
<td>
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
</td>
</tr>
</tbody>
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0,
32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","<p>&nbsp;</p>
<table>
<tbody>
<tr>
<td>
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
</td>
</tr>
</tbody>
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0,
32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0,
32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`;
//utils.analysestring(txtstring)
//console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR')
// .split("\n")[0].split(","))
utils.csv2json = (csv, options, callback) => {
// EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT
// l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché
console.log("\n--------------- CSV2JSON ---------------\n");
// Default CSV options
if (!options.retln) options.retln = "\n";
if (csv.indexOf("\n\r") > -1) options.retln = "\n\r";
if (!options.sep) options.sep = ";";
//gestion d un separateur dans une chaine de texte
//const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm');
//csv = csv.replace(regseptext, "CARACSEPAR");
// csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE")
csv = utils.replacecarbtweendblquote(csv, options.sep, "CARSEPARATOR");
if (!options.replacespecialcarCsv2Json) {
options.replacespecialcarCsv2Json = [];
} else {
if (typeof options.replacespecialcarCsv2Json == "string") {
//permet de passer des regex en string
options.replacespecialcarCsv2Json = eval(
options.replacespecialcarCsv2Json
);
}
}
const result = [];
const lines = csv.split(options.retln);
const headers = utils.getHeaders(lines, options.sep);
let unknownHeaders = "";
//console.log('headers', headers)
//console.log('options.champs', options.champs)
headers.forEach((header) => {
// Si un header n'est pas présent dans la liste des champs prédéfinis
// on l'ajoute aux champs inconnus
if (options.champs.indexOf(header) === -1) {
unknownHeaders += `${header}, `;
}
});
if (unknownHeaders !== "") {
const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`;
return callback(errorMsg, null);
}
lines.forEach((line, index) => {
// Skip headers line or empty lines
if (index === 0 || line.replace(/\s/g, "").length === 0) {
return;
}
// pour debuguer on met origincsv pour voir la ligne d'origine
const currentLineData = { origincsv: line, linenumber: index };
const currentLine = line.split(options.sep); // Current string in the line
for (let j = 0; j < headers.length; j++) {
// Si la ligne n'est pas vide
if (currentLine[j]) {
// On clean le champs
// ajout eventuel de modification de caracter reservé ; dans les libelléetc...
let currentValue = currentLine[j].trim();
//on transforme le caractere separateur modifié entre double quote
currentValue = currentValue.replace("CARSEPARATOR", options.sep);
options.replacespecialcarCsv2Json.forEach((re) => {
currentValue = currentValue.replace(re[0], re[1]);
});
// Si le header est un email
if (headers[j].includes("EMAIL")) {
// Supprimer tous les espaces
currentValue = currentLine[j].replace(/\s/g, "");
}
// on check si le chamos doit être numerique
if (options.numericfield.includes(headers[j])) {
currentValue = currentLine[j].replace(/\,/g, ".");
try {
const test = parseFloat(currentValue);
} catch (er) {
return callback(
`${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`,
null
);
}
}
if (currentValue) {
// Si le header actuel est de type array
// Cela signifie que le header apparaît plusieurs fois dans le CSV
// et que les valeurs correspondantes à ce header
// doivent être mis dans un array
if (options.array && options.array.indexOf(headers[j]) > -1) {
// Si le tableau pour ce header n'existe pas on le crée
if (!currentLineData[headers[j]]) {
currentLineData[headers[j]] = [];
}
if (options.arraysplitsep) {
currentValue.split(options.arraysplitsep).forEach((v) => {
currentLineData[headers[j]].push(v);
});
} else {
currentLineData[headers[j]].push(currentValue);
}
} else {
// Si un header est déjà présent pour la ligne
// alors que il n'est pas spécifié comme étant un array
// on retourne une erreur
if (currentLineData[headers[j]]) {
const errorMsg = `Le champ ${headers[j]} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`;
return callback(errorMsg, null);
}
currentLineData[headers[j]] = currentValue;
}
}
}
}
result.push(currentLineData);
});
return callback(null, result);
};
/**
* [csvparam2json description]
* @param {object} csv object of csv file that has been read
* @param {object} options object containing csv options, headers, ...
{retln:'code de retour de ligne \n ou \n\r',
sep:'code to split cells',
champs:[ch1,ch2,...] catch only those field,
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
* @param {Function} callback callback function
* @return {callback} - return an error if error, else return json
it converts a csv with 3 column col1;col2;col3 in a json in a tree
if in col1 we have __ => then it splits a leaf
col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored
return data = {xxxx:{yyyy:value}}
col1 = xxxx; col2 = value; col3 = comment ignored
return data = {xxxx:value}
Usage example:
fiche.csvparam2article = (err, fiche) => {
if (!err) {
console.log(fiche)
}
}
utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
retln: "\n",
sep: ";",
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
}, fiche.csv2article)
*/
utils.csvparam2json = (csv, options, callback) => {
console.log("\n--------------- CSVPARAM2JSON ---------------\n");
let etat = "";
const param = {};
if (!options.retln) {
options.retln = "\n";
}
if (csv.indexOf("\n\r") > -1) {
options.retln = "\n\r";
}
if (!options.sep) {
options.sep = ";";
}
if (!options.seplevel) {
options.seplevel = "__";
}
if (!options.replacespecialcarCsv2Json) {
options.replacespecialcarCsv2Json = [];
} else {
if (typeof options.replacespecialcarCsv2Json == "string") {
//permet de passer des regex en string
options.replacespecialcarCsv2Json = eval(
options.replacespecialcarCsv2Json
);
}
}
const lines = csv.split(options.retln);
for (let i = 0; i < lines.length; i++) {
const infol = lines[i].split(options.sep);
//console.log(infol)
if (infol[0].length > 4 && infol.length < 2) {
// si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb
etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`;
continue;
}
// On ajoute ici la gestion de tous les caracteres spéciaux
// reservées pour le csv ; ' etc..'
if (infol[1] && infol[1] + "" == infol[1]) {
options.replacespecialcarCsv2Json.forEach((re) => {
//console.log("gggggggggggggggggggg", infol[1])
infol[1] = infol[1].replace(re[0], re[1]);
});
// console.log(infol[1])
infol[1] = infol[1].replace(/'|/g, '"');
//console.log(infol[1])
if (infol[1].toLowerCase() === "true") {
infol[1] = true;
} else if (infol[1].toLowerCase() === "false") {
infol[1] = false;
}
}
console.log(infol[1]);
//supprime des lignes vides
if (infol[0] == "") continue;
if (infol[0].indexOf(options.seplevel) == -1) {
param[infol[0]] = infol[1];
continue;
} else {
const arbre = infol[0].split(options.seplevel);
switch (arbre.length) {
case 1:
param[arbre[0]] = infol[1];
break;
case 2:
if (arbre[1] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
param[arbre[0]][arbre[1]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = [];
//console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"'))
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]].push(result);
}
break;
case 3:
if (arbre[2] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
param[arbre[0]][arbre[1]][arbre[2]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = [];
//eval("result = \"test\"");
//console.log(result);
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]][arbre[1]].push(result);
}
break;
case 4:
if (arbre[3] != "ARRAY") {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
if (!param[arbre[0]][arbre[1]][arbre[2]])
param[arbre[0]][arbre[1]][arbre[2]] = {};
param[arbre[0]][arbre[1]][arbre[2]][arbre[3]] = infol[1];
} else {
if (!param[arbre[0]]) param[arbre[0]] = {};
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
if (!param[arbre[0]][arbre[1]][arbre[2]])
param[arbre[0]][arbre[1]][arbre[2]] = [];
eval("result=" + infol[1]);
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
param[arbre[0]][arbre[1]][arbre[2]].push(result);
break;
}
default:
break;
}
}
}
// JSON.parse(JSON.stringify(param))
console.log(
"kkkkkkkkkkkkkkkkkk",
param["catalogue"]["filtrecatalog"]["searchengine"]
);
if (etat == "") {
return callback(null, JSON.parse(JSON.stringify(param)));
} else {
return callback(etat, null);
}
};
utils.levenshtein = (a, b) => {
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
let tmp, i, j, prev, val, row;
// swap to save some memory O(min(a,b)) instead of O(a)
if (a.length > b.length) {
tmp = a;
a = b;
b = tmp;
}
row = Array(a.length + 1);
// init the row
for (i = 0; i <= a.length; i++) {
row[i] = i;
}
// fill in the rest
for (i = 1; i <= b.length; i++) {
prev = i;
for (j = 1; j <= a.length; j++) {
if (b[i - 1] === a[j - 1]) {
val = row[j - 1]; // match
} else {
val = Math.min(
row[j - 1] + 1, // substitution
Math.min(
prev + 1, // insertion
row[j] + 1
)
); // deletion
}
row[j - 1] = prev;
prev = val;
}
row[a.length] = prev;
}
return row[a.length];
};
utils.testinarray = (array, arrayreferent) => {
// au moins un element de array existe dans arryreferent
let exist = false;
if (arrayreferent) {
//console.log('arrrrrrrrrrrrrrr', arrayreferent)
array.forEach((e) => {
//console.log(e)
if (arrayreferent.includes(e)) exist = true;
});
}
return exist;
};
/*
DIRECTORY
*/
const isDirectory = (source) => fs.lstatSync(source).isDirectory();
const getDirectories = (source) =>
fs
.readdirSync(source)
.map((name) => path.join(source, name))
.filter(isDirectory);
module.exports = utils;