/* eslint-disable no-useless-escape */ const fs = require("fs"); const path = require("path"); const bcrypt = require("bcrypt"); const moment = require("moment"); const utils = {}; console.log( "Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apxtrib project (see /package.json to get email).\n We'll add to the roadmap to add it." ); /** * EMAIL */ /* const validateEmail = email => { const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; return regExp.test(email); }; const validatePassword = pwd => { const regExp = new RegExp( /^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/ ); return regExp.test(pwd); }; const filterInvalidInArray = (array, validate) => array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid /** * POSTAL CODE */ /* const validatePostalCode = postalCode => /(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode); /** * PHONE */ /* const validatePhoneNumber = phoneNumber => /((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber); const correctPhoneNumber = phone => phone[0] === '0' ? '+33' + phone.substr(1) : phone; const Checkjson = (appProfil, referential, data) => { // @TODO get a referentiel per object then check data validity and allowed access // need to add referentiel manager const invalidefor = []; let updateDatabase = false; Object.keys(data).forEach(field => { switch (field) { case 'token': updateDatabase = true; break; case 'email': if (!validateEmail(data.email)) { invalidefor.push('ERREMAIL:' + field); } else { updateDatabase = true; } break; case 'password': if (!validatePassword(data.password)) { invalidefor.push('ERRPWD:' + field); } else { data.password = bcrypt.hash(data.password, config.saltRounds); updateDatabase = true; } break; } }); return { invalidefor, data, updateDatabase }; }; */ //Permet d'attendre en milliseconde // s'utilise avec async ()=>{ // await sleep(2000) //} utils.sleep = (ms) => { return new Promise((resolve) => setTimeout(resolve, ms)); }; utils.normalize = {}; utils.normalize.telephonefr = (phone) => { phone = phone.trim().replace(/[- .]/g, ""); if ( Checkjson.schema.properties.format.telephoenfr(phone) && phone.length == 10 && phone[0] == "0" ) { phone = "+33 " + phone.substring(1); } return phone; }; utils.normalize.zfill10 = (num) => { let s = num + ""; while (s.length < 10) s = "0" + s; return s; }; utils.generemdp = (nbpos, fromchar) => { if (!fromchar) { const fromchar = "ABCDEFGHIJKLMNPQRSTUVWZY123456789"; } //const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789"; let mdp = ""; for (var i = 0; i < nbpos; i++) { var pos = Math.floor(Math.random() * fromchar.length); mdp += fromchar.substring(pos, pos + 1); } return mdp; }; utils.generecompteur = (filecpt, typeincrement) => { let file = `${filecpt}/${typeincrement}.json`; let prefix = ""; if ((typeincrement = "ANNEESEMAINE")) { file = `${filecpt}/${typeincrement}${moment().format( "YYYY" )}${moment().format("WW")}.json`; prefix = `${moment().format("YYYY")}${moment().format("WW")}`; } let num = 1; try { num = parseInt(fs.readFileSync(file, "utf8")) + 1; } catch (err) { console.log("Nouveau compteur incrementale ", file); } fs.writeFileSync(file, num, "utf8"); return prefix + num; }; /** * CSV */ utils.json2csv = (jsondata, options, callback) => { // uniquement json = [{niv1:val,niv1:[liste of val]}] // console.log('_________________________'); // console.log(jsondata) // console.log('_________________________'); if (jsondata.length == 0) { return callback("Empty json", null); } if (!options.retln) options.retln = "\n"; if (!options.sep) options.sep = ";"; if (!options.arraysplitsep) options.arraysplitsep = ","; if (!options.replacespecialcarJson2Csv) { options.replacespecialcarJson2Csv = []; } else { if (typeof options.replacespecialcarJson2Csv == "string") { //permet de passer des regex en string options.replacespecialcarJson2Csv = eval( options.replacespecialcarJson2Csv ); } } let etat = ""; let csv = ""; let entete = ""; let prem = true; for (const j in jsondata) { // console.log(jsondata[j]) for (const c in options.champs) { if (prem) { entete += options.champs[c] + options.sep; } if (jsondata[j][options.champs[c]]) { if (options.array.indexOf(options.champs[c]) > -1) { csv += jsondata[j][options.champs[c]].join(options.arraysplitsep) + options.sep; } else { let currentValue = ""; if (jsondata[j][options.champs[c]]) currentValue += jsondata[j][options.champs[c]]; options.replacespecialcarJson2Csv.forEach((re) => { //console.log(currentValue) currentValue = currentValue.replace(re[1], re[0]); }); csv += currentValue + options.sep; } } else { csv += options.sep; } } csv = csv.substring(0, csv.length - 1) + options.retln; if (prem) { prem = false; entete = entete.substring(0, entete.length - 1) + options.retln; // console.log(entete) } } // return entete + csv; if (etat == "") { return callback(null, entete + csv); } else { return callback(etat, null); } }; /** * Get headers from first line of CSV * @param {array} lines array of string which contains each csv lines * @return {array} string array of headers */ utils.getHeaders = (lines, sep) => lines[0].split(sep).map((i) => i.replace(/"/g, "")); /** * [csv2json description] * @param {object} csv object of csv file that has been read * @param {object} options object containing csv options, headers, ... {retln:'code de retour de ligne \n ou \n\r', sep:'code to split cells', champs:[ch1,ch2,...] catch only those field, array:[ch1, ] can have more than one field champs with same name then data are push into an array } * @param {Function} callback callback function * @return {callback} - return an error if error, else return json it convert a csv file into a json = [{field:value}] Usage example: fiche.csv2article = (err, fiche) => { if (!err) { console.log(fiche) } } utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), { retln: "\n", sep: ";", champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"], array: ["TAG", "PHONE_PRO", "EMAIL_PRO"] }, fiche.csv2article) */ utils.replacecarbtweendblquote = (csv, car, carremplacant) => { /* return csv text with any car betwenn 2 " by CARSEPARATOR */ let newcsv = ""; let txtencours = ""; let flagouvert = false; const sepreg = new RegExp(`${car}`, "gmi"); for (let j = 0; j < csv.length; j++) { //if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) { if (csv[j] == '"') { if (flagouvert) { // on cherche à ferme une chaine de texte if (csv[j + 1] == '"') { //on a "" consecutif qu'on remplace par "" et on fait j+1 txtencours += '""'; j++; } else { // on a bien une fermeture flagouvert = false; newcsv += txtencours.replace(sepreg, carremplacant); txtencours = '"'; } } else { // on ouvre une chaine flagouvert = true; //on met le contenu précédent ds newcsv newcsv += txtencours; txtencours = '"'; } //} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) { } else if (csv[j] !== "\n") { txtencours += csv[j]; // } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) { } else if (csv[j] == "\n") { if (!flagouvert) txtencours += "\n"; } } return newcsv + txtencours; }; utils.analysestring = (string) => { let buftxt = ""; let bufcode = ""; let i = 0; let avecRL = false; for (let p = 0; p < string.length; p++) { if (string[p].charCodeAt() == 10) { buftxt += "[RL]"; avecRL = true; } else { buftxt += string[p]; } bufcode += "-" + string[p].charCodeAt(); if (i == 20) { if (avecRL) { console.log(`${buftxt} - ${bufcode}`); } else { console.log(`${buftxt} ---- ${bufcode}`); } i = 0; buftxt = ""; bufcode = ""; avecRL = false; } i++; } }; const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","
Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking |
Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking |
PACK JOB SEARCH
",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0, 32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","PACK JOB SEARCH
",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`; //utils.analysestring(txtstring) //console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR') // .split("\n")[0].split(",")) utils.csv2json = (csv, options, callback) => { // EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT // l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché console.log("\n--------------- CSV2JSON ---------------\n"); // Default CSV options if (!options.retln) options.retln = "\n"; if (csv.indexOf("\n\r") > -1) options.retln = "\n\r"; if (!options.sep) options.sep = ";"; //gestion d un separateur dans une chaine de texte //const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm'); //csv = csv.replace(regseptext, "CARACSEPAR"); // csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE") csv = utils.replacecarbtweendblquote(csv, options.sep, "CARSEPARATOR"); if (!options.replacespecialcarCsv2Json) { options.replacespecialcarCsv2Json = []; } else { if (typeof options.replacespecialcarCsv2Json == "string") { //permet de passer des regex en string options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json ); } } const result = []; const lines = csv.split(options.retln); const headers = utils.getHeaders(lines, options.sep); let unknownHeaders = ""; //console.log('headers', headers) //console.log('options.champs', options.champs) headers.forEach((header) => { // Si un header n'est pas présent dans la liste des champs prédéfinis // on l'ajoute aux champs inconnus if (options.champs.indexOf(header) === -1) { unknownHeaders += `${header}, `; } }); if (unknownHeaders !== "") { const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`; return callback(errorMsg, null); } lines.forEach((line, index) => { // Skip headers line or empty lines if (index === 0 || line.replace(/\s/g, "").length === 0) { return; } // pour debuguer on met origincsv pour voir la ligne d'origine const currentLineData = { origincsv: line, linenumber: index }; const currentLine = line.split(options.sep); // Current string in the line for (let j = 0; j < headers.length; j++) { // Si la ligne n'est pas vide if (currentLine[j]) { // On clean le champs // ajout eventuel de modification de caracter reservé ; dans les libelléetc... let currentValue = currentLine[j].trim(); //on transforme le caractere separateur modifié entre double quote currentValue = currentValue.replace("CARSEPARATOR", options.sep); options.replacespecialcarCsv2Json.forEach((re) => { currentValue = currentValue.replace(re[0], re[1]); }); // Si le header est un email if (headers[j].includes("EMAIL")) { // Supprimer tous les espaces currentValue = currentLine[j].replace(/\s/g, ""); } // on check si le chamos doit être numerique if (options.numericfield.includes(headers[j])) { currentValue = currentLine[j].replace(/\,/g, "."); try { const test = parseFloat(currentValue); } catch (er) { return callback( `${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`, null ); } } if (currentValue) { // Si le header actuel est de type array // Cela signifie que le header apparaît plusieurs fois dans le CSV // et que les valeurs correspondantes à ce header // doivent être mis dans un array if (options.array && options.array.indexOf(headers[j]) > -1) { // Si le tableau pour ce header n'existe pas on le crée if (!currentLineData[headers[j]]) { currentLineData[headers[j]] = []; } if (options.arraysplitsep) { currentValue.split(options.arraysplitsep).forEach((v) => { currentLineData[headers[j]].push(v); }); } else { currentLineData[headers[j]].push(currentValue); } } else { // Si un header est déjà présent pour la ligne // alors que il n'est pas spécifié comme étant un array // on retourne une erreur if (currentLineData[headers[j]]) { const errorMsg = `Le champ ${headers[j]} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`; return callback(errorMsg, null); } currentLineData[headers[j]] = currentValue; } } } } result.push(currentLineData); }); return callback(null, result); }; /** * [csvparam2json description] * @param {object} csv object of csv file that has been read * @param {object} options object containing csv options, headers, ... {retln:'code de retour de ligne \n ou \n\r', sep:'code to split cells', champs:[ch1,ch2,...] catch only those field, array:[ch1, ] can have more than one field champs with same name then data are push into an array } * @param {Function} callback callback function * @return {callback} - return an error if error, else return json it converts a csv with 3 column col1;col2;col3 in a json in a tree if in col1 we have __ => then it splits a leaf col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored return data = {xxxx:{yyyy:value}} col1 = xxxx; col2 = value; col3 = comment ignored return data = {xxxx:value} Usage example: fiche.csvparam2article = (err, fiche) => { if (!err) { console.log(fiche) } } utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), { retln: "\n", sep: ";", champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"], array: ["TAG", "PHONE_PRO", "EMAIL_PRO"] }, fiche.csv2article) */ utils.csvparam2json = (csv, options, callback) => { console.log("\n--------------- CSVPARAM2JSON ---------------\n"); let etat = ""; const param = {}; if (!options.retln) { options.retln = "\n"; } if (csv.indexOf("\n\r") > -1) { options.retln = "\n\r"; } if (!options.sep) { options.sep = ";"; } if (!options.seplevel) { options.seplevel = "__"; } if (!options.replacespecialcarCsv2Json) { options.replacespecialcarCsv2Json = []; } else { if (typeof options.replacespecialcarCsv2Json == "string") { //permet de passer des regex en string options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json ); } } const lines = csv.split(options.retln); for (let i = 0; i < lines.length; i++) { const infol = lines[i].split(options.sep); //console.log(infol) if (infol[0].length > 4 && infol.length < 2) { // si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`; continue; } // On ajoute ici la gestion de tous les caracteres spéciaux // reservées pour le csv ; ' etc..' if (infol[1] && infol[1] + "" == infol[1]) { options.replacespecialcarCsv2Json.forEach((re) => { //console.log("gggggggggggggggggggg", infol[1]) infol[1] = infol[1].replace(re[0], re[1]); }); // console.log(infol[1]) infol[1] = infol[1].replace(/'|’/g, '"'); //console.log(infol[1]) if (infol[1].toLowerCase() === "true") { infol[1] = true; } else if (infol[1].toLowerCase() === "false") { infol[1] = false; } } console.log(infol[1]); //supprime des lignes vides if (infol[0] == "") continue; if (infol[0].indexOf(options.seplevel) == -1) { param[infol[0]] = infol[1]; continue; } else { const arbre = infol[0].split(options.seplevel); switch (arbre.length) { case 1: param[arbre[0]] = infol[1]; break; case 2: if (arbre[1] != "ARRAY") { if (!param[arbre[0]]) param[arbre[0]] = {}; param[arbre[0]][arbre[1]] = infol[1]; } else { if (!param[arbre[0]]) param[arbre[0]] = []; //console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"')) eval("result=" + infol[1]); //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[arbre[0]].push(result); } break; case 3: if (arbre[2] != "ARRAY") { if (!param[arbre[0]]) param[arbre[0]] = {}; if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {}; param[arbre[0]][arbre[1]][arbre[2]] = infol[1]; } else { if (!param[arbre[0]]) param[arbre[0]] = {}; if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = []; //eval("result = \"test\""); //console.log(result); eval("result=" + infol[1]); //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[arbre[0]][arbre[1]].push(result); } break; case 4: if (arbre[3] != "ARRAY") { if (!param[arbre[0]]) param[arbre[0]] = {}; if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {}; if (!param[arbre[0]][arbre[1]][arbre[2]]) param[arbre[0]][arbre[1]][arbre[2]] = {}; param[arbre[0]][arbre[1]][arbre[2]][arbre[3]] = infol[1]; } else { if (!param[arbre[0]]) param[arbre[0]] = {}; if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {}; if (!param[arbre[0]][arbre[1]][arbre[2]]) param[arbre[0]][arbre[1]][arbre[2]] = []; eval("result=" + infol[1]); //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[arbre[0]][arbre[1]][arbre[2]].push(result); break; } default: break; } } } // JSON.parse(JSON.stringify(param)) console.log( "kkkkkkkkkkkkkkkkkk", param["catalogue"]["filtrecatalog"]["searchengine"] ); if (etat == "") { return callback(null, JSON.parse(JSON.stringify(param))); } else { return callback(etat, null); } }; utils.levenshtein = (a, b) => { if (a.length === 0) return b.length; if (b.length === 0) return a.length; let tmp, i, j, prev, val, row; // swap to save some memory O(min(a,b)) instead of O(a) if (a.length > b.length) { tmp = a; a = b; b = tmp; } row = Array(a.length + 1); // init the row for (i = 0; i <= a.length; i++) { row[i] = i; } // fill in the rest for (i = 1; i <= b.length; i++) { prev = i; for (j = 1; j <= a.length; j++) { if (b[i - 1] === a[j - 1]) { val = row[j - 1]; // match } else { val = Math.min( row[j - 1] + 1, // substitution Math.min( prev + 1, // insertion row[j] + 1 ) ); // deletion } row[j - 1] = prev; prev = val; } row[a.length] = prev; } return row[a.length]; }; utils.testinarray = (array, arrayreferent) => { // au moins un element de array existe dans arryreferent let exist = false; if (arrayreferent) { //console.log('arrrrrrrrrrrrrrr', arrayreferent) array.forEach((e) => { //console.log(e) if (arrayreferent.includes(e)) exist = true; }); } return exist; }; /* DIRECTORY */ const isDirectory = (source) => fs.lstatSync(source).isDirectory(); const getDirectories = (source) => fs .readdirSync(source) .map((name) => path.join(source, name)) .filter(isDirectory); module.exports = utils;