647 lines
22 KiB
JavaScript
647 lines
22 KiB
JavaScript
|
/* eslint-disable no-useless-escape */
|
|||
|
|
const fs = require("fs");
|
|||
|
|
const path = require("path");
|
|||
|
|
const bcrypt = require("bcrypt");
|
|||
|
|
const moment = require("moment");
|
|||
|
|
const utils = {};
|
|||
|
|
|
|||
|
|
console.log(
|
|||
|
|
"Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apxtrib project (see /package.json to get email).\n We'll add to the roadmap to add it."
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* EMAIL
|
|||
|
|
*/
|
|||
|
|
/* const validateEmail = email => {
|
|||
|
|
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
|
|||
|
|
return regExp.test(email);
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const validatePassword = pwd => {
|
|||
|
|
const regExp = new RegExp(
|
|||
|
|
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
|
|||
|
|
);
|
|||
|
|
return regExp.test(pwd);
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const filterInvalidInArray = (array, validate) =>
|
|||
|
|
array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* POSTAL CODE
|
|||
|
|
*/
|
|||
|
|
/*
|
|||
|
|
const validatePostalCode = postalCode =>
|
|||
|
|
/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode);
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* PHONE
|
|||
|
|
*/
|
|||
|
|
/* const validatePhoneNumber = phoneNumber =>
|
|||
|
|
/((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber);
|
|||
|
|
|
|||
|
|
const correctPhoneNumber = phone =>
|
|||
|
|
phone[0] === '0' ? '+33' + phone.substr(1) : phone;
|
|||
|
|
|
|||
|
|
const Checkjson = (appProfil, referential, data) => {
|
|||
|
|
// @TODO get a referentiel per object then check data validity and allowed access
|
|||
|
|
// need to add referentiel manager
|
|||
|
|
const invalidefor = [];
|
|||
|
|
let updateDatabase = false;
|
|||
|
|
Object.keys(data).forEach(field => {
|
|||
|
|
switch (field) {
|
|||
|
|
case 'token':
|
|||
|
|
updateDatabase = true;
|
|||
|
|
break;
|
|||
|
|
case 'email':
|
|||
|
|
if (!validateEmail(data.email)) {
|
|||
|
|
invalidefor.push('ERREMAIL:' + field);
|
|||
|
|
} else {
|
|||
|
|
updateDatabase = true;
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 'password':
|
|||
|
|
if (!validatePassword(data.password)) {
|
|||
|
|
invalidefor.push('ERRPWD:' + field);
|
|||
|
|
} else {
|
|||
|
|
data.password = bcrypt.hash(data.password, config.saltRounds);
|
|||
|
|
updateDatabase = true;
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
return { invalidefor, data, updateDatabase };
|
|||
|
|
};
|
|||
|
|
*/
|
|||
|
|
//Permet d'attendre en milliseconde
|
|||
|
|
// s'utilise avec async ()=>{
|
|||
|
|
// await sleep(2000)
|
|||
|
|
//}
|
|||
|
|
utils.sleep = (ms) => {
|
|||
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|||
|
|
};
|
|||
|
|
utils.normalize = {};
|
|||
|
|
utils.normalize.telephonefr = (phone) => {
|
|||
|
|
phone = phone.trim().replace(/[- .]/g, "");
|
|||
|
|
if (
|
|||
|
|
Checkjson.schema.properties.format.telephoenfr(phone) &&
|
|||
|
|
phone.length == 10 &&
|
|||
|
|
phone[0] == "0"
|
|||
|
|
) {
|
|||
|
|
phone = "+33 " + phone.substring(1);
|
|||
|
|
}
|
|||
|
|
return phone;
|
|||
|
|
};
|
|||
|
|
utils.normalize.zfill10 = (num) => {
|
|||
|
|
let s = num + "";
|
|||
|
|
while (s.length < 10) s = "0" + s;
|
|||
|
|
return s;
|
|||
|
|
};
|
|||
|
|
utils.generemdp = (nbpos, fromchar) => {
|
|||
|
|
if (!fromchar) {
|
|||
|
|
const fromchar = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
|
|||
|
|
}
|
|||
|
|
//const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
|
|||
|
|
let mdp = "";
|
|||
|
|
for (var i = 0; i < nbpos; i++) {
|
|||
|
|
var pos = Math.floor(Math.random() * fromchar.length);
|
|||
|
|
mdp += fromchar.substring(pos, pos + 1);
|
|||
|
|
}
|
|||
|
|
return mdp;
|
|||
|
|
};
|
|||
|
|
utils.generecompteur = (filecpt, typeincrement) => {
|
|||
|
|
let file = `${filecpt}/${typeincrement}.json`;
|
|||
|
|
let prefix = "";
|
|||
|
|
if ((typeincrement = "ANNEESEMAINE")) {
|
|||
|
|
file = `${filecpt}/${typeincrement}${moment().format(
|
|||
|
|
"YYYY"
|
|||
|
|
)}${moment().format("WW")}.json`;
|
|||
|
|
prefix = `${moment().format("YYYY")}${moment().format("WW")}`;
|
|||
|
|
}
|
|||
|
|
let num = 1;
|
|||
|
|
try {
|
|||
|
|
num = parseInt(fs.readFileSync(file, "utf8")) + 1;
|
|||
|
|
} catch (err) {
|
|||
|
|
console.log("Nouveau compteur incrementale ", file);
|
|||
|
|
}
|
|||
|
|
fs.writeFileSync(file, num, "utf8");
|
|||
|
|
return prefix + num;
|
|||
|
|
};
|
|||
|
|
/**
|
|||
|
|
* CSV
|
|||
|
|
*/
|
|||
|
|
utils.json2csv = (jsondata, options, callback) => {
|
|||
|
|
// uniquement json = [{niv1:val,niv1:[liste of val]}]
|
|||
|
|
// console.log('_________________________');
|
|||
|
|
// console.log(jsondata)
|
|||
|
|
// console.log('_________________________');
|
|||
|
|
if (jsondata.length == 0) {
|
|||
|
|
return callback("Empty json", null);
|
|||
|
|
}
|
|||
|
|
if (!options.retln) options.retln = "\n";
|
|||
|
|
if (!options.sep) options.sep = ";";
|
|||
|
|
if (!options.arraysplitsep) options.arraysplitsep = ",";
|
|||
|
|
if (!options.replacespecialcarJson2Csv) {
|
|||
|
|
options.replacespecialcarJson2Csv = [];
|
|||
|
|
} else {
|
|||
|
|
if (typeof options.replacespecialcarJson2Csv == "string") {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarJson2Csv = eval(
|
|||
|
|
options.replacespecialcarJson2Csv
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
let etat = "";
|
|||
|
|
let csv = "";
|
|||
|
|
let entete = "";
|
|||
|
|
let prem = true;
|
|||
|
|
for (const j in jsondata) {
|
|||
|
|
// console.log(jsondata[j])
|
|||
|
|
for (const c in options.champs) {
|
|||
|
|
if (prem) {
|
|||
|
|
entete += options.champs[c] + options.sep;
|
|||
|
|
}
|
|||
|
|
if (jsondata[j][options.champs[c]]) {
|
|||
|
|
if (options.array.indexOf(options.champs[c]) > -1) {
|
|||
|
|
csv +=
|
|||
|
|
jsondata[j][options.champs[c]].join(options.arraysplitsep) +
|
|||
|
|
options.sep;
|
|||
|
|
} else {
|
|||
|
|
let currentValue = "";
|
|||
|
|
if (jsondata[j][options.champs[c]])
|
|||
|
|
currentValue += jsondata[j][options.champs[c]];
|
|||
|
|
options.replacespecialcarJson2Csv.forEach((re) => {
|
|||
|
|
//console.log(currentValue)
|
|||
|
|
currentValue = currentValue.replace(re[1], re[0]);
|
|||
|
|
});
|
|||
|
|
csv += currentValue + options.sep;
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
csv += options.sep;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
csv = csv.substring(0, csv.length - 1) + options.retln;
|
|||
|
|
if (prem) {
|
|||
|
|
prem = false;
|
|||
|
|
entete = entete.substring(0, entete.length - 1) + options.retln;
|
|||
|
|
// console.log(entete)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// return entete + csv;
|
|||
|
|
if (etat == "") {
|
|||
|
|
return callback(null, entete + csv);
|
|||
|
|
} else {
|
|||
|
|
return callback(etat, null);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
/**
|
|||
|
|
* Get headers from first line of CSV
|
|||
|
|
* @param {array} lines array of string which contains each csv lines
|
|||
|
|
* @return {array} string array of headers
|
|||
|
|
*/
|
|||
|
|
utils.getHeaders = (lines, sep) =>
|
|||
|
|
lines[0].split(sep).map((i) => i.replace(/"/g, ""));
|
|||
|
|
/**
|
|||
|
|
* [csv2json description]
|
|||
|
|
* @param {object} csv object of csv file that has been read
|
|||
|
|
* @param {object} options object containing csv options, headers, ...
|
|||
|
|
{retln:'code de retour de ligne \n ou \n\r',
|
|||
|
|
sep:'code to split cells',
|
|||
|
|
champs:[ch1,ch2,...] catch only those field,
|
|||
|
|
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
|||
|
|
* @param {Function} callback callback function
|
|||
|
|
* @return {callback} - return an error if error, else return json
|
|||
|
|
it convert a csv file into a json = [{field:value}]
|
|||
|
|
|
|||
|
|
Usage example:
|
|||
|
|
fiche.csv2article = (err, fiche) => {
|
|||
|
|
if (!err) {
|
|||
|
|
console.log(fiche)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
|||
|
|
retln: "\n",
|
|||
|
|
sep: ";",
|
|||
|
|
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
|||
|
|
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
|||
|
|
}, fiche.csv2article)
|
|||
|
|
|
|||
|
|
*/
|
|||
|
|
utils.replacecarbtweendblquote = (csv, car, carremplacant) => {
|
|||
|
|
/*
|
|||
|
|
return csv text with any car betwenn 2 " by CARSEPARATOR
|
|||
|
|
*/
|
|||
|
|
let newcsv = "";
|
|||
|
|
let txtencours = "";
|
|||
|
|
let flagouvert = false;
|
|||
|
|
const sepreg = new RegExp(`${car}`, "gmi");
|
|||
|
|
for (let j = 0; j < csv.length; j++) {
|
|||
|
|
//if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) {
|
|||
|
|
if (csv[j] == '"') {
|
|||
|
|
if (flagouvert) {
|
|||
|
|
// on cherche à ferme une chaine de texte
|
|||
|
|
if (csv[j + 1] == '"') {
|
|||
|
|
//on a "" consecutif qu'on remplace par "" et on fait j+1
|
|||
|
|
txtencours += '""';
|
|||
|
|
j++;
|
|||
|
|
} else {
|
|||
|
|
// on a bien une fermeture
|
|||
|
|
flagouvert = false;
|
|||
|
|
newcsv += txtencours.replace(sepreg, carremplacant);
|
|||
|
|
txtencours = '"';
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// on ouvre une chaine
|
|||
|
|
flagouvert = true;
|
|||
|
|
//on met le contenu précédent ds newcsv
|
|||
|
|
newcsv += txtencours;
|
|||
|
|
txtencours = '"';
|
|||
|
|
}
|
|||
|
|
//} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) {
|
|||
|
|
} else if (csv[j] !== "\n") {
|
|||
|
|
txtencours += csv[j];
|
|||
|
|
// } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) {
|
|||
|
|
} else if (csv[j] == "\n") {
|
|||
|
|
if (!flagouvert) txtencours += "\n";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return newcsv + txtencours;
|
|||
|
|
};
|
|||
|
|
utils.analysestring = (string) => {
|
|||
|
|
let buftxt = "";
|
|||
|
|
let bufcode = "";
|
|||
|
|
let i = 0;
|
|||
|
|
let avecRL = false;
|
|||
|
|
for (let p = 0; p < string.length; p++) {
|
|||
|
|
if (string[p].charCodeAt() == 10) {
|
|||
|
|
buftxt += "[RL]";
|
|||
|
|
avecRL = true;
|
|||
|
|
} else {
|
|||
|
|
buftxt += string[p];
|
|||
|
|
}
|
|||
|
|
bufcode += "-" + string[p].charCodeAt();
|
|||
|
|
if (i == 20) {
|
|||
|
|
if (avecRL) {
|
|||
|
|
console.log(`${buftxt} - ${bufcode}`);
|
|||
|
|
} else {
|
|||
|
|
console.log(`${buftxt} ---- ${bufcode}`);
|
|||
|
|
}
|
|||
|
|
i = 0;
|
|||
|
|
buftxt = "";
|
|||
|
|
bufcode = "";
|
|||
|
|
avecRL = false;
|
|||
|
|
}
|
|||
|
|
i++;
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","<p> </p>
|
|||
|
|
<table>
|
|||
|
|
<tbody>
|
|||
|
|
<tr>
|
|||
|
|
<td>
|
|||
|
|
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
|||
|
|
</td>
|
|||
|
|
</tr>
|
|||
|
|
</tbody>
|
|||
|
|
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0,
|
|||
|
|
32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","<p> </p>
|
|||
|
|
<table>
|
|||
|
|
<tbody>
|
|||
|
|
<tr>
|
|||
|
|
<td>
|
|||
|
|
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
|||
|
|
</td>
|
|||
|
|
</tr>
|
|||
|
|
</tbody>
|
|||
|
|
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0,
|
|||
|
|
32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0,
|
|||
|
|
32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`;
|
|||
|
|
//utils.analysestring(txtstring)
|
|||
|
|
//console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR')
|
|||
|
|
// .split("\n")[0].split(","))
|
|||
|
|
utils.csv2json = (csv, options, callback) => {
|
|||
|
|
// EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT
|
|||
|
|
// l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché
|
|||
|
|
console.log("\n--------------- CSV2JSON ---------------\n");
|
|||
|
|
// Default CSV options
|
|||
|
|
if (!options.retln) options.retln = "\n";
|
|||
|
|
if (csv.indexOf("\n\r") > -1) options.retln = "\n\r";
|
|||
|
|
if (!options.sep) options.sep = ";";
|
|||
|
|
//gestion d un separateur dans une chaine de texte
|
|||
|
|
//const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm');
|
|||
|
|
//csv = csv.replace(regseptext, "CARACSEPAR");
|
|||
|
|
// csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE")
|
|||
|
|
csv = utils.replacecarbtweendblquote(csv, options.sep, "CARSEPARATOR");
|
|||
|
|
if (!options.replacespecialcarCsv2Json) {
|
|||
|
|
options.replacespecialcarCsv2Json = [];
|
|||
|
|
} else {
|
|||
|
|
if (typeof options.replacespecialcarCsv2Json == "string") {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarCsv2Json = eval(
|
|||
|
|
options.replacespecialcarCsv2Json
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
const result = [];
|
|||
|
|
const lines = csv.split(options.retln);
|
|||
|
|
const headers = utils.getHeaders(lines, options.sep);
|
|||
|
|
let unknownHeaders = "";
|
|||
|
|
//console.log('headers', headers)
|
|||
|
|
//console.log('options.champs', options.champs)
|
|||
|
|
headers.forEach((header) => {
|
|||
|
|
// Si un header n'est pas présent dans la liste des champs prédéfinis
|
|||
|
|
// on l'ajoute aux champs inconnus
|
|||
|
|
if (options.champs.indexOf(header) === -1) {
|
|||
|
|
unknownHeaders += `${header}, `;
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
if (unknownHeaders !== "") {
|
|||
|
|
const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`;
|
|||
|
|
return callback(errorMsg, null);
|
|||
|
|
}
|
|||
|
|
lines.forEach((line, index) => {
|
|||
|
|
// Skip headers line or empty lines
|
|||
|
|
if (index === 0 || line.replace(/\s/g, "").length === 0) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
// pour debuguer on met origincsv pour voir la ligne d'origine
|
|||
|
|
const currentLineData = { origincsv: line, linenumber: index };
|
|||
|
|
const currentLine = line.split(options.sep); // Current string in the line
|
|||
|
|
for (let j = 0; j < headers.length; j++) {
|
|||
|
|
// Si la ligne n'est pas vide
|
|||
|
|
if (currentLine[j]) {
|
|||
|
|
// On clean le champs
|
|||
|
|
// ajout eventuel de modification de caracter reservé ; dans les libelléetc...
|
|||
|
|
let currentValue = currentLine[j].trim();
|
|||
|
|
//on transforme le caractere separateur modifié entre double quote
|
|||
|
|
currentValue = currentValue.replace("CARSEPARATOR", options.sep);
|
|||
|
|
options.replacespecialcarCsv2Json.forEach((re) => {
|
|||
|
|
currentValue = currentValue.replace(re[0], re[1]);
|
|||
|
|
});
|
|||
|
|
// Si le header est un email
|
|||
|
|
if (headers[j].includes("EMAIL")) {
|
|||
|
|
// Supprimer tous les espaces
|
|||
|
|
currentValue = currentLine[j].replace(/\s/g, "");
|
|||
|
|
}
|
|||
|
|
// on check si le chamos doit être numerique
|
|||
|
|
if (options.numericfield.includes(headers[j])) {
|
|||
|
|
currentValue = currentLine[j].replace(/\,/g, ".");
|
|||
|
|
try {
|
|||
|
|
const test = parseFloat(currentValue);
|
|||
|
|
} catch (er) {
|
|||
|
|
return callback(
|
|||
|
|
`${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`,
|
|||
|
|
null
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (currentValue) {
|
|||
|
|
// Si le header actuel est de type array
|
|||
|
|
// Cela signifie que le header apparaît plusieurs fois dans le CSV
|
|||
|
|
// et que les valeurs correspondantes à ce header
|
|||
|
|
// doivent être mis dans un array
|
|||
|
|
if (options.array && options.array.indexOf(headers[j]) > -1) {
|
|||
|
|
// Si le tableau pour ce header n'existe pas on le crée
|
|||
|
|
if (!currentLineData[headers[j]]) {
|
|||
|
|
currentLineData[headers[j]] = [];
|
|||
|
|
}
|
|||
|
|
if (options.arraysplitsep) {
|
|||
|
|
currentValue.split(options.arraysplitsep).forEach((v) => {
|
|||
|
|
currentLineData[headers[j]].push(v);
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
currentLineData[headers[j]].push(currentValue);
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// Si un header est déjà présent pour la ligne
|
|||
|
|
// alors que il n'est pas spécifié comme étant un array
|
|||
|
|
// on retourne une erreur
|
|||
|
|
if (currentLineData[headers[j]]) {
|
|||
|
|
const errorMsg = `Le champ ${headers[j]} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`;
|
|||
|
|
return callback(errorMsg, null);
|
|||
|
|
}
|
|||
|
|
currentLineData[headers[j]] = currentValue;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
result.push(currentLineData);
|
|||
|
|
});
|
|||
|
|
return callback(null, result);
|
|||
|
|
};
|
|||
|
|
/**
|
|||
|
|
* [csvparam2json description]
|
|||
|
|
* @param {object} csv object of csv file that has been read
|
|||
|
|
* @param {object} options object containing csv options, headers, ...
|
|||
|
|
{retln:'code de retour de ligne \n ou \n\r',
|
|||
|
|
sep:'code to split cells',
|
|||
|
|
champs:[ch1,ch2,...] catch only those field,
|
|||
|
|
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
|||
|
|
* @param {Function} callback callback function
|
|||
|
|
* @return {callback} - return an error if error, else return json
|
|||
|
|
it converts a csv with 3 column col1;col2;col3 in a json in a tree
|
|||
|
|
if in col1 we have __ => then it splits a leaf
|
|||
|
|
col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored
|
|||
|
|
return data = {xxxx:{yyyy:value}}
|
|||
|
|
col1 = xxxx; col2 = value; col3 = comment ignored
|
|||
|
|
return data = {xxxx:value}
|
|||
|
|
|
|||
|
|
Usage example:
|
|||
|
|
fiche.csvparam2article = (err, fiche) => {
|
|||
|
|
if (!err) {
|
|||
|
|
console.log(fiche)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
|||
|
|
retln: "\n",
|
|||
|
|
sep: ";",
|
|||
|
|
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
|||
|
|
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
|||
|
|
}, fiche.csv2article)
|
|||
|
|
|
|||
|
|
*/
|
|||
|
|
utils.csvparam2json = (csv, options, callback) => {
|
|||
|
|
console.log("\n--------------- CSVPARAM2JSON ---------------\n");
|
|||
|
|
let etat = "";
|
|||
|
|
const param = {};
|
|||
|
|
if (!options.retln) {
|
|||
|
|
options.retln = "\n";
|
|||
|
|
}
|
|||
|
|
if (csv.indexOf("\n\r") > -1) {
|
|||
|
|
options.retln = "\n\r";
|
|||
|
|
}
|
|||
|
|
if (!options.sep) {
|
|||
|
|
options.sep = ";";
|
|||
|
|
}
|
|||
|
|
if (!options.seplevel) {
|
|||
|
|
options.seplevel = "__";
|
|||
|
|
}
|
|||
|
|
if (!options.replacespecialcarCsv2Json) {
|
|||
|
|
options.replacespecialcarCsv2Json = [];
|
|||
|
|
} else {
|
|||
|
|
if (typeof options.replacespecialcarCsv2Json == "string") {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarCsv2Json = eval(
|
|||
|
|
options.replacespecialcarCsv2Json
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
const lines = csv.split(options.retln);
|
|||
|
|
for (let i = 0; i < lines.length; i++) {
|
|||
|
|
const infol = lines[i].split(options.sep);
|
|||
|
|
//console.log(infol)
|
|||
|
|
if (infol[0].length > 4 && infol.length < 2) {
|
|||
|
|
// si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb
|
|||
|
|
etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
// On ajoute ici la gestion de tous les caracteres spéciaux
|
|||
|
|
// reservées pour le csv ; ' etc..'
|
|||
|
|
if (infol[1] && infol[1] + "" == infol[1]) {
|
|||
|
|
options.replacespecialcarCsv2Json.forEach((re) => {
|
|||
|
|
//console.log("gggggggggggggggggggg", infol[1])
|
|||
|
|
infol[1] = infol[1].replace(re[0], re[1]);
|
|||
|
|
});
|
|||
|
|
// console.log(infol[1])
|
|||
|
|
infol[1] = infol[1].replace(/'|’/g, '"');
|
|||
|
|
//console.log(infol[1])
|
|||
|
|
if (infol[1].toLowerCase() === "true") {
|
|||
|
|
infol[1] = true;
|
|||
|
|
} else if (infol[1].toLowerCase() === "false") {
|
|||
|
|
infol[1] = false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
console.log(infol[1]);
|
|||
|
|
//supprime des lignes vides
|
|||
|
|
if (infol[0] == "") continue;
|
|||
|
|
if (infol[0].indexOf(options.seplevel) == -1) {
|
|||
|
|
param[infol[0]] = infol[1];
|
|||
|
|
continue;
|
|||
|
|
} else {
|
|||
|
|
const arbre = infol[0].split(options.seplevel);
|
|||
|
|
switch (arbre.length) {
|
|||
|
|
case 1:
|
|||
|
|
param[arbre[0]] = infol[1];
|
|||
|
|
break;
|
|||
|
|
case 2:
|
|||
|
|
if (arbre[1] != "ARRAY") {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = {};
|
|||
|
|
param[arbre[0]][arbre[1]] = infol[1];
|
|||
|
|
} else {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = [];
|
|||
|
|
//console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
eval("result=" + infol[1]);
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[arbre[0]].push(result);
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 3:
|
|||
|
|
if (arbre[2] != "ARRAY") {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
|
|||
|
|
param[arbre[0]][arbre[1]][arbre[2]] = infol[1];
|
|||
|
|
} else {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = [];
|
|||
|
|
//eval("result = \"test\"");
|
|||
|
|
//console.log(result);
|
|||
|
|
eval("result=" + infol[1]);
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[arbre[0]][arbre[1]].push(result);
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 4:
|
|||
|
|
if (arbre[3] != "ARRAY") {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]][arbre[2]])
|
|||
|
|
param[arbre[0]][arbre[1]][arbre[2]] = {};
|
|||
|
|
param[arbre[0]][arbre[1]][arbre[2]][arbre[3]] = infol[1];
|
|||
|
|
} else {
|
|||
|
|
if (!param[arbre[0]]) param[arbre[0]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]]) param[arbre[0]][arbre[1]] = {};
|
|||
|
|
if (!param[arbre[0]][arbre[1]][arbre[2]])
|
|||
|
|
param[arbre[0]][arbre[1]][arbre[2]] = [];
|
|||
|
|
eval("result=" + infol[1]);
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[arbre[0]][arbre[1]][arbre[2]].push(result);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
default:
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// JSON.parse(JSON.stringify(param))
|
|||
|
|
console.log(
|
|||
|
|
"kkkkkkkkkkkkkkkkkk",
|
|||
|
|
param["catalogue"]["filtrecatalog"]["searchengine"]
|
|||
|
|
);
|
|||
|
|
if (etat == "") {
|
|||
|
|
return callback(null, JSON.parse(JSON.stringify(param)));
|
|||
|
|
} else {
|
|||
|
|
return callback(etat, null);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
utils.levenshtein = (a, b) => {
|
|||
|
|
if (a.length === 0) return b.length;
|
|||
|
|
if (b.length === 0) return a.length;
|
|||
|
|
let tmp, i, j, prev, val, row;
|
|||
|
|
// swap to save some memory O(min(a,b)) instead of O(a)
|
|||
|
|
if (a.length > b.length) {
|
|||
|
|
tmp = a;
|
|||
|
|
a = b;
|
|||
|
|
b = tmp;
|
|||
|
|
}
|
|||
|
|
row = Array(a.length + 1);
|
|||
|
|
// init the row
|
|||
|
|
for (i = 0; i <= a.length; i++) {
|
|||
|
|
row[i] = i;
|
|||
|
|
}
|
|||
|
|
// fill in the rest
|
|||
|
|
for (i = 1; i <= b.length; i++) {
|
|||
|
|
prev = i;
|
|||
|
|
for (j = 1; j <= a.length; j++) {
|
|||
|
|
if (b[i - 1] === a[j - 1]) {
|
|||
|
|
val = row[j - 1]; // match
|
|||
|
|
} else {
|
|||
|
|
val = Math.min(
|
|||
|
|
row[j - 1] + 1, // substitution
|
|||
|
|
Math.min(
|
|||
|
|
prev + 1, // insertion
|
|||
|
|
row[j] + 1
|
|||
|
|
)
|
|||
|
|
); // deletion
|
|||
|
|
}
|
|||
|
|
row[j - 1] = prev;
|
|||
|
|
prev = val;
|
|||
|
|
}
|
|||
|
|
row[a.length] = prev;
|
|||
|
|
}
|
|||
|
|
return row[a.length];
|
|||
|
|
};
|
|||
|
|
utils.testinarray = (array, arrayreferent) => {
|
|||
|
|
// au moins un element de array existe dans arryreferent
|
|||
|
|
let exist = false;
|
|||
|
|
if (arrayreferent) {
|
|||
|
|
//console.log('arrrrrrrrrrrrrrr', arrayreferent)
|
|||
|
|
array.forEach((e) => {
|
|||
|
|
//console.log(e)
|
|||
|
|
if (arrayreferent.includes(e)) exist = true;
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
return exist;
|
|||
|
|
};
|
|||
|
|
/*
|
|||
|
|
DIRECTORY
|
|||
|
|
*/
|
|||
|
|
const isDirectory = (source) => fs.lstatSync(source).isDirectory();
|
|||
|
|
const getDirectories = (source) =>
|
|||
|
|
fs
|
|||
|
|
.readdirSync(source)
|
|||
|
|
.map((name) => path.join(source, name))
|
|||
|
|
.filter(isDirectory);
|
|||
|
|
|
|||
|
|
module.exports = utils;
|