forked from apxtri/apxtrib
605 lines
21 KiB
JavaScript
605 lines
21 KiB
JavaScript
|
/* eslint-disable no-useless-escape */
|
|||
|
|
const fs = require( 'fs' );
|
|||
|
|
const path = require( 'path' );
|
|||
|
|
const bcrypt = require( 'bcrypt' );
|
|||
|
|
const moment = require( 'moment' );
|
|||
|
|
const config = require( '../config' );
|
|||
|
|
const utils = {};
|
|||
|
|
|
|||
|
|
console.log( "Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apixtribe project (see /package.json to get email).\n We'll add to the roadmap to add it." );
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* EMAIL
|
|||
|
|
*/
|
|||
|
|
/* const validateEmail = email => {
|
|||
|
|
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
|
|||
|
|
return regExp.test(email);
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const validatePassword = pwd => {
|
|||
|
|
const regExp = new RegExp(
|
|||
|
|
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
|
|||
|
|
);
|
|||
|
|
return regExp.test(pwd);
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const filterInvalidInArray = (array, validate) =>
|
|||
|
|
array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* POSTAL CODE
|
|||
|
|
*/
|
|||
|
|
/*
|
|||
|
|
const validatePostalCode = postalCode =>
|
|||
|
|
/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode);
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* PHONE
|
|||
|
|
*/
|
|||
|
|
/* const validatePhoneNumber = phoneNumber =>
|
|||
|
|
/((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber);
|
|||
|
|
|
|||
|
|
const correctPhoneNumber = phone =>
|
|||
|
|
phone[0] === '0' ? '+33' + phone.substr(1) : phone;
|
|||
|
|
|
|||
|
|
const checkData = (appProfil, referential, data) => {
|
|||
|
|
// @TODO get a referentiel per object then check data validity and allowed access
|
|||
|
|
// need to add referentiel manager
|
|||
|
|
const invalidefor = [];
|
|||
|
|
let updateDatabase = false;
|
|||
|
|
Object.keys(data).forEach(field => {
|
|||
|
|
switch (field) {
|
|||
|
|
case 'token':
|
|||
|
|
updateDatabase = true;
|
|||
|
|
break;
|
|||
|
|
case 'email':
|
|||
|
|
if (!validateEmail(data.email)) {
|
|||
|
|
invalidefor.push('ERREMAIL:' + field);
|
|||
|
|
} else {
|
|||
|
|
updateDatabase = true;
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 'password':
|
|||
|
|
if (!validatePassword(data.password)) {
|
|||
|
|
invalidefor.push('ERRPWD:' + field);
|
|||
|
|
} else {
|
|||
|
|
data.password = bcrypt.hash(data.password, config.saltRounds);
|
|||
|
|
updateDatabase = true;
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
return { invalidefor, data, updateDatabase };
|
|||
|
|
};
|
|||
|
|
*/
|
|||
|
|
//Permet d'attendre en milliseconde
|
|||
|
|
// s'utilise avec async ()=>{
|
|||
|
|
// await sleep(2000)
|
|||
|
|
//}
|
|||
|
|
utils.sleep = ( ms ) => {
|
|||
|
|
return new Promise( resolve => setTimeout( resolve, ms ) );
|
|||
|
|
}
|
|||
|
|
utils.generemdp = ( nbpos ) => {
|
|||
|
|
const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
|
|||
|
|
let mdp = "";
|
|||
|
|
for( var i = 0; i < nbpos; i++ ) {
|
|||
|
|
var pos = Math.floor( Math.random() * chaine.length );
|
|||
|
|
mdp += chaine.substring( pos, pos + 1 );
|
|||
|
|
}
|
|||
|
|
return mdp;
|
|||
|
|
}
|
|||
|
|
utils.generecompteur = ( filecpt, typeincrement ) => {
|
|||
|
|
let file = `${filecpt}/${typeincrement}.json`;
|
|||
|
|
let prefix = "";
|
|||
|
|
if( typeincrement = "ANNEESEMAINE" ) {
|
|||
|
|
file = `${filecpt}/${typeincrement}${moment().format('YYYY')}${moment().format('WW')}.json`
|
|||
|
|
prefix = `${moment().format('YYYY')}${moment().format('WW')}`
|
|||
|
|
}
|
|||
|
|
let num = 1;
|
|||
|
|
try {
|
|||
|
|
num = parseInt( fs.readFileSync( file, 'utf8' ) ) + 1;
|
|||
|
|
} catch ( err ) {
|
|||
|
|
console.log( "Nouveau compteur incrementale ", file )
|
|||
|
|
}
|
|||
|
|
fs.writeFileSync( file, num, 'utf8' );
|
|||
|
|
return prefix + num
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* CSV
|
|||
|
|
*/
|
|||
|
|
utils.json2csv = ( jsondata, options, callback ) => {
|
|||
|
|
// uniquement json = [{niv1:val,niv1:[liste of val]}]
|
|||
|
|
// console.log('_________________________');
|
|||
|
|
// console.log(jsondata)
|
|||
|
|
// console.log('_________________________');
|
|||
|
|
if( jsondata.length == 0 ) {
|
|||
|
|
return callback( "Empty json", null );
|
|||
|
|
}
|
|||
|
|
if( !options.retln ) options.retln = '\n';
|
|||
|
|
if( !options.sep ) options.sep = ';';
|
|||
|
|
if( !options.arraysplitsep ) options.arraysplitsep = ",";
|
|||
|
|
if( !options.replacespecialcarJson2Csv ) {
|
|||
|
|
options.replacespecialcarJson2Csv = []
|
|||
|
|
} else {
|
|||
|
|
if( typeof options.replacespecialcarJson2Csv == "string" ) {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarJson2Csv = eval( options.replacespecialcarJson2Csv );
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
let etat = "";
|
|||
|
|
let csv = '';
|
|||
|
|
let entete = '';
|
|||
|
|
let prem = true;
|
|||
|
|
for( const j in jsondata ) {
|
|||
|
|
// console.log(jsondata[j])
|
|||
|
|
for( const c in options.champs ) {
|
|||
|
|
if( prem ) {
|
|||
|
|
entete += options.champs[ c ] + options.sep;
|
|||
|
|
}
|
|||
|
|
if( jsondata[ j ][ options.champs[ c ] ] ) {
|
|||
|
|
if( options.array.indexOf( options.champs[ c ] ) > -1 ) {
|
|||
|
|
csv += jsondata[ j ][ options.champs[ c ] ].join( options.arraysplitsep ) + options.sep;
|
|||
|
|
} else {
|
|||
|
|
let currentValue = "";
|
|||
|
|
if( jsondata[ j ][ options.champs[ c ] ] ) currentValue += jsondata[ j ][ options.champs[ c ] ];
|
|||
|
|
options.replacespecialcarJson2Csv.forEach( re => {
|
|||
|
|
//console.log(currentValue)
|
|||
|
|
currentValue = currentValue.replace( re[ 1 ], re[ 0 ] )
|
|||
|
|
} )
|
|||
|
|
csv += currentValue + options.sep;
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
csv += options.sep;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
csv = csv.substring( 0, csv.length - 1 ) + options.retln;
|
|||
|
|
if( prem ) {
|
|||
|
|
prem = false;
|
|||
|
|
entete = entete.substring( 0, entete.length - 1 ) + options.retln;
|
|||
|
|
// console.log(entete)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// return entete + csv;
|
|||
|
|
if( etat == "" ) {
|
|||
|
|
return callback( null, entete + csv );
|
|||
|
|
} else {
|
|||
|
|
return callback( etat, null );
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
/**
|
|||
|
|
* Get headers from first line of CSV
|
|||
|
|
* @param {array} lines array of string which contains each csv lines
|
|||
|
|
* @return {array} string array of headers
|
|||
|
|
*/
|
|||
|
|
utils.getHeaders = ( lines, sep ) => lines[ 0 ].split( sep )
|
|||
|
|
.map( i => i.replace( /"/g, '' ) );
|
|||
|
|
/**
|
|||
|
|
* [csv2json description]
|
|||
|
|
* @param {object} csv object of csv file that has been read
|
|||
|
|
* @param {object} options object containing csv options, headers, ...
|
|||
|
|
{retln:'code de retour de ligne \n ou \n\r',
|
|||
|
|
sep:'code to split cells',
|
|||
|
|
champs:[ch1,ch2,...] catch only those field,
|
|||
|
|
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
|||
|
|
* @param {Function} callback callback function
|
|||
|
|
* @return {callback} - return an error if error, else return json
|
|||
|
|
it convert a csv file into a json = [{field:value}]
|
|||
|
|
|
|||
|
|
Usage example:
|
|||
|
|
fiche.csv2article = (err, fiche) => {
|
|||
|
|
if (!err) {
|
|||
|
|
console.log(fiche)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
|||
|
|
retln: "\n",
|
|||
|
|
sep: ";",
|
|||
|
|
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
|||
|
|
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
|||
|
|
}, fiche.csv2article)
|
|||
|
|
|
|||
|
|
*/
|
|||
|
|
utils.replacecarbtweendblquote = ( csv, car, carremplacant ) => {
|
|||
|
|
/*
|
|||
|
|
return csv text with any car betwenn 2 " by CARSEPARATOR
|
|||
|
|
*/
|
|||
|
|
let newcsv = "";
|
|||
|
|
let txtencours = "";
|
|||
|
|
let flagouvert = false
|
|||
|
|
const sepreg = new RegExp( `${car}`, 'gmi' )
|
|||
|
|
for( let j = 0; j < csv.length; j++ ) {
|
|||
|
|
//if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) {
|
|||
|
|
if( csv[ j ] == "\"" ) {
|
|||
|
|
if( flagouvert ) {
|
|||
|
|
// on cherche à ferme une chaine de texte
|
|||
|
|
if( csv[ j + 1 ] == "\"" ) {
|
|||
|
|
//on a "" consecutif qu'on remplace par "" et on fait j+1
|
|||
|
|
txtencours += "\"\""
|
|||
|
|
j++
|
|||
|
|
} else {
|
|||
|
|
// on a bien une fermeture
|
|||
|
|
flagouvert = false
|
|||
|
|
newcsv += txtencours.replace( sepreg, carremplacant )
|
|||
|
|
txtencours = "\""
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// on ouvre une chaine
|
|||
|
|
flagouvert = true
|
|||
|
|
//on met le contenu précédent ds newcsv
|
|||
|
|
newcsv += txtencours
|
|||
|
|
txtencours = "\""
|
|||
|
|
}
|
|||
|
|
//} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) {
|
|||
|
|
} else if( csv[ j ] !== "\n" ) {
|
|||
|
|
txtencours += csv[ j ]
|
|||
|
|
// } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) {
|
|||
|
|
} else if( csv[ j ] == "\n" ) {
|
|||
|
|
if( !flagouvert ) txtencours += "\n"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return newcsv + txtencours
|
|||
|
|
}
|
|||
|
|
utils.analysestring = ( string ) => {
|
|||
|
|
let buftxt = ""
|
|||
|
|
let bufcode = ""
|
|||
|
|
let i = 0
|
|||
|
|
let avecRL = false
|
|||
|
|
for( let p = 0; p < string.length; p++ ) {
|
|||
|
|
if( string[ p ].charCodeAt() == 10 ) {
|
|||
|
|
buftxt += "[RL]"
|
|||
|
|
avecRL = true
|
|||
|
|
} else {
|
|||
|
|
buftxt += string[ p ]
|
|||
|
|
}
|
|||
|
|
bufcode += "-" + string[ p ].charCodeAt();
|
|||
|
|
if( i == 20 ) {
|
|||
|
|
if( avecRL ) {
|
|||
|
|
console.log( `${buftxt} - ${bufcode}` )
|
|||
|
|
} else {
|
|||
|
|
console.log( `${buftxt} ---- ${bufcode}` )
|
|||
|
|
}
|
|||
|
|
i = 0;
|
|||
|
|
buftxt = ""
|
|||
|
|
bufcode = ""
|
|||
|
|
avecRL = false
|
|||
|
|
}
|
|||
|
|
i++;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","<p> </p>
|
|||
|
|
<table>
|
|||
|
|
<tbody>
|
|||
|
|
<tr>
|
|||
|
|
<td>
|
|||
|
|
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
|||
|
|
</td>
|
|||
|
|
</tr>
|
|||
|
|
</tbody>
|
|||
|
|
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0,
|
|||
|
|
32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","<p> </p>
|
|||
|
|
<table>
|
|||
|
|
<tbody>
|
|||
|
|
<tr>
|
|||
|
|
<td>
|
|||
|
|
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
|||
|
|
</td>
|
|||
|
|
</tr>
|
|||
|
|
</tbody>
|
|||
|
|
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0,
|
|||
|
|
32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0,
|
|||
|
|
32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`
|
|||
|
|
//utils.analysestring(txtstring)
|
|||
|
|
//console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR')
|
|||
|
|
// .split("\n")[0].split(","))
|
|||
|
|
utils.csv2json = ( csv, options, callback ) => {
|
|||
|
|
// EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT
|
|||
|
|
// l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché
|
|||
|
|
console.log( '\n--------------- CSV2JSON ---------------\n' );
|
|||
|
|
// Default CSV options
|
|||
|
|
if( !options.retln ) options.retln = '\n';
|
|||
|
|
if( csv.indexOf( '\n\r' ) > -1 ) options.retln = '\n\r';
|
|||
|
|
if( !options.sep ) options.sep = ';';
|
|||
|
|
//gestion d un separateur dans une chaine de texte
|
|||
|
|
//const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm');
|
|||
|
|
//csv = csv.replace(regseptext, "CARACSEPAR");
|
|||
|
|
// csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE")
|
|||
|
|
csv = utils.replacecarbtweendblquote( csv, options.sep, "CARSEPARATOR" )
|
|||
|
|
if( !options.replacespecialcarCsv2Json ) {
|
|||
|
|
options.replacespecialcarCsv2Json = []
|
|||
|
|
} else {
|
|||
|
|
if( typeof options.replacespecialcarCsv2Json == "string" ) {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json );
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
const result = [];
|
|||
|
|
const lines = csv.split( options.retln );
|
|||
|
|
const headers = utils.getHeaders( lines, options.sep );
|
|||
|
|
let unknownHeaders = '';
|
|||
|
|
//console.log('headers', headers)
|
|||
|
|
//console.log('options.champs', options.champs)
|
|||
|
|
headers.forEach( header => {
|
|||
|
|
// Si un header n'est pas présent dans la liste des champs prédéfinis
|
|||
|
|
// on l'ajoute aux champs inconnus
|
|||
|
|
if( options.champs.indexOf( header ) === -1 ) {
|
|||
|
|
unknownHeaders += `${header}, `;
|
|||
|
|
}
|
|||
|
|
} );
|
|||
|
|
if( unknownHeaders !== '' ) {
|
|||
|
|
const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`;
|
|||
|
|
return callback( errorMsg, null );
|
|||
|
|
}
|
|||
|
|
lines.forEach( ( line, index ) => {
|
|||
|
|
// Skip headers line or empty lines
|
|||
|
|
if( index === 0 || line.replace( /\s/g, '' )
|
|||
|
|
.length === 0 ) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
// pour debuguer on met origincsv pour voir la ligne d'origine
|
|||
|
|
const currentLineData = { 'origincsv': line, 'linenumber': index };
|
|||
|
|
const currentLine = line.split( options.sep ); // Current string in the line
|
|||
|
|
for( let j = 0; j < headers.length; j++ ) {
|
|||
|
|
// Si la ligne n'est pas vide
|
|||
|
|
if( currentLine[ j ] ) {
|
|||
|
|
// On clean le champs
|
|||
|
|
// ajout eventuel de modification de caracter reservé ; dans les libelléetc...
|
|||
|
|
let currentValue = currentLine[ j ].trim()
|
|||
|
|
//on transforme le caractere separateur modifié entre double quote
|
|||
|
|
currentValue = currentValue.replace( 'CARSEPARATOR', options.sep );
|
|||
|
|
options.replacespecialcarCsv2Json.forEach( re => {
|
|||
|
|
currentValue = currentValue.replace( re[ 0 ], re[ 1 ] )
|
|||
|
|
} )
|
|||
|
|
// Si le header est un email
|
|||
|
|
if( headers[ j ].includes( 'EMAIL' ) ) {
|
|||
|
|
// Supprimer tous les espaces
|
|||
|
|
currentValue = currentLine[ j ].replace( /\s/g, '' );
|
|||
|
|
}
|
|||
|
|
// on check si le chamos doit être numerique
|
|||
|
|
if( options.numericfield.includes( headers[ j ] ) ) {
|
|||
|
|
currentValue = currentLine[ j ].replace( /\,/g, '.' );
|
|||
|
|
try {
|
|||
|
|
const test = parseFloat( currentValue );
|
|||
|
|
} catch ( er ) {
|
|||
|
|
return callback( `${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`, null );
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if( currentValue ) {
|
|||
|
|
// Si le header actuel est de type array
|
|||
|
|
// Cela signifie que le header apparaît plusieurs fois dans le CSV
|
|||
|
|
// et que les valeurs correspondantes à ce header
|
|||
|
|
// doivent être mis dans un array
|
|||
|
|
if( options.array && options.array.indexOf( headers[ j ] ) > -1 ) {
|
|||
|
|
// Si le tableau pour ce header n'existe pas on le crée
|
|||
|
|
if( !currentLineData[ headers[ j ] ] ) {
|
|||
|
|
currentLineData[ headers[ j ] ] = [];
|
|||
|
|
}
|
|||
|
|
if( options.arraysplitsep ) {
|
|||
|
|
currentValue.split( options.arraysplitsep )
|
|||
|
|
.forEach( v => {
|
|||
|
|
currentLineData[ headers[ j ] ].push( v );
|
|||
|
|
} )
|
|||
|
|
} else {
|
|||
|
|
currentLineData[ headers[ j ] ].push( currentValue );
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// Si un header est déjà présent pour la ligne
|
|||
|
|
// alors que il n'est pas spécifié comme étant un array
|
|||
|
|
// on retourne une erreur
|
|||
|
|
if( currentLineData[ headers[ j ] ] ) {
|
|||
|
|
const errorMsg = `Le champ ${
|
|||
|
|
headers[j]
|
|||
|
|
} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`;
|
|||
|
|
return callback( errorMsg, null );
|
|||
|
|
}
|
|||
|
|
currentLineData[ headers[ j ] ] = currentValue;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
result.push( currentLineData );
|
|||
|
|
} );
|
|||
|
|
return callback( null, result );
|
|||
|
|
};
|
|||
|
|
/**
|
|||
|
|
* [csvparam2json description]
|
|||
|
|
* @param {object} csv object of csv file that has been read
|
|||
|
|
* @param {object} options object containing csv options, headers, ...
|
|||
|
|
{retln:'code de retour de ligne \n ou \n\r',
|
|||
|
|
sep:'code to split cells',
|
|||
|
|
champs:[ch1,ch2,...] catch only those field,
|
|||
|
|
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
|||
|
|
* @param {Function} callback callback function
|
|||
|
|
* @return {callback} - return an error if error, else return json
|
|||
|
|
it converts a csv with 3 column col1;col2;col3 in a json in a tree
|
|||
|
|
if in col1 we have __ => then it splits a leaf
|
|||
|
|
col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored
|
|||
|
|
return data = {xxxx:{yyyy:value}}
|
|||
|
|
col1 = xxxx; col2 = value; col3 = comment ignored
|
|||
|
|
return data = {xxxx:value}
|
|||
|
|
|
|||
|
|
Usage example:
|
|||
|
|
fiche.csvparam2article = (err, fiche) => {
|
|||
|
|
if (!err) {
|
|||
|
|
console.log(fiche)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
|||
|
|
retln: "\n",
|
|||
|
|
sep: ";",
|
|||
|
|
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
|||
|
|
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
|||
|
|
}, fiche.csv2article)
|
|||
|
|
|
|||
|
|
*/
|
|||
|
|
utils.csvparam2json = ( csv, options, callback ) => {
|
|||
|
|
console.log( '\n--------------- CSVPARAM2JSON ---------------\n' );
|
|||
|
|
let etat = "";
|
|||
|
|
const param = {};
|
|||
|
|
if( !options.retln ) {
|
|||
|
|
options.retln = '\n';
|
|||
|
|
}
|
|||
|
|
if( csv.indexOf( '\n\r' ) > -1 ) {
|
|||
|
|
options.retln = '\n\r';
|
|||
|
|
}
|
|||
|
|
if( !options.sep ) {
|
|||
|
|
options.sep = ';';
|
|||
|
|
}
|
|||
|
|
if( !options.seplevel ) {
|
|||
|
|
options.seplevel = "__";
|
|||
|
|
}
|
|||
|
|
if( !options.replacespecialcarCsv2Json ) {
|
|||
|
|
options.replacespecialcarCsv2Json = []
|
|||
|
|
} else {
|
|||
|
|
if( typeof options.replacespecialcarCsv2Json == "string" ) {
|
|||
|
|
//permet de passer des regex en string
|
|||
|
|
options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json );
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
const lines = csv.split( options.retln );
|
|||
|
|
for( let i = 0; i < lines.length; i++ ) {
|
|||
|
|
const infol = lines[ i ].split( options.sep )
|
|||
|
|
//console.log(infol)
|
|||
|
|
if( infol[ 0 ].length > 4 && infol.length < 2 ) {
|
|||
|
|
// si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb
|
|||
|
|
etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
// On ajoute ici la gestion de tous les caracteres spéciaux
|
|||
|
|
// reservées pour le csv ; ' etc..'
|
|||
|
|
if( infol[ 1 ] && infol[ 1 ] + "" == infol[ 1 ] ) {
|
|||
|
|
options.replacespecialcarCsv2Json.forEach( re => {
|
|||
|
|
//console.log("gggggggggggggggggggg", infol[1])
|
|||
|
|
infol[ 1 ] = infol[ 1 ].replace( re[ 0 ], re[ 1 ] );
|
|||
|
|
} )
|
|||
|
|
// console.log(infol[1])
|
|||
|
|
infol[ 1 ] = infol[ 1 ].replace( /'|’/g, "\"" );
|
|||
|
|
//console.log(infol[1])
|
|||
|
|
if( infol[ 1 ].toLowerCase() === 'true' ) {
|
|||
|
|
infol[ 1 ] = true;
|
|||
|
|
} else if( infol[ 1 ].toLowerCase() === 'false' ) {
|
|||
|
|
infol[ 1 ] = false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
console.log( infol[ 1 ] )
|
|||
|
|
//supprime des lignes vides
|
|||
|
|
if( infol[ 0 ] == '' ) continue;
|
|||
|
|
if( infol[ 0 ].indexOf( options.seplevel ) == -1 ) {
|
|||
|
|
param[ infol[ 0 ] ] = infol[ 1 ]
|
|||
|
|
continue;
|
|||
|
|
} else {
|
|||
|
|
const arbre = infol[ 0 ].split( options.seplevel )
|
|||
|
|
switch ( arbre.length ) {
|
|||
|
|
case 1:
|
|||
|
|
param[ arbre[ 0 ] ] = infol[ 1 ];
|
|||
|
|
break;
|
|||
|
|
case 2:
|
|||
|
|
if( arbre[ 1 ] != "ARRAY" ) {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
|||
|
|
param[ arbre[ 0 ] ][ arbre[ 1 ] ] = infol[ 1 ];
|
|||
|
|
} else {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = [];
|
|||
|
|
//console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
eval( "result=" + infol[ 1 ] )
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[ arbre[ 0 ] ].push( result )
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 3:
|
|||
|
|
if( arbre[ 2 ] != "ARRAY" ) {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
|||
|
|
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = infol[ 1 ];
|
|||
|
|
} else {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = [];
|
|||
|
|
//eval("result = \"test\"");
|
|||
|
|
//console.log(result);
|
|||
|
|
eval( "result=" + infol[ 1 ] );
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[ arbre[ 0 ] ][ arbre[ 1 ] ].push( result )
|
|||
|
|
}
|
|||
|
|
break;
|
|||
|
|
case 4:
|
|||
|
|
if( arbre[ 3 ] != "ARRAY" ) {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = {};
|
|||
|
|
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ][ arbre[ 3 ] ] = infol[ 1 ];
|
|||
|
|
} else {
|
|||
|
|
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
|||
|
|
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = [];
|
|||
|
|
eval( "result=" + infol[ 1 ] )
|
|||
|
|
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
|||
|
|
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ].push( result )
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
default:
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// JSON.parse(JSON.stringify(param))
|
|||
|
|
console.log( 'kkkkkkkkkkkkkkkkkk', param[ 'catalogue' ][ 'filtrecatalog' ][ 'searchengine' ] )
|
|||
|
|
if( etat == "" ) {
|
|||
|
|
return callback( null, JSON.parse( JSON.stringify( param ) ) );
|
|||
|
|
} else {
|
|||
|
|
return callback( etat, null );
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
utils.levenshtein = ( a, b ) => {
|
|||
|
|
if( a.length === 0 ) return b.length;
|
|||
|
|
if( b.length === 0 ) return a.length;
|
|||
|
|
let tmp, i, j, prev, val, row;
|
|||
|
|
// swap to save some memory O(min(a,b)) instead of O(a)
|
|||
|
|
if( a.length > b.length ) {
|
|||
|
|
tmp = a;
|
|||
|
|
a = b;
|
|||
|
|
b = tmp;
|
|||
|
|
}
|
|||
|
|
row = Array( a.length + 1 );
|
|||
|
|
// init the row
|
|||
|
|
for( i = 0; i <= a.length; i++ ) {
|
|||
|
|
row[ i ] = i;
|
|||
|
|
}
|
|||
|
|
// fill in the rest
|
|||
|
|
for( i = 1; i <= b.length; i++ ) {
|
|||
|
|
prev = i;
|
|||
|
|
for( j = 1; j <= a.length; j++ ) {
|
|||
|
|
if( b[ i - 1 ] === a[ j - 1 ] ) {
|
|||
|
|
val = row[ j - 1 ]; // match
|
|||
|
|
} else {
|
|||
|
|
val = Math.min( row[ j - 1 ] + 1, // substitution
|
|||
|
|
Math.min( prev + 1, // insertion
|
|||
|
|
row[ j ] + 1 ) ); // deletion
|
|||
|
|
}
|
|||
|
|
row[ j - 1 ] = prev;
|
|||
|
|
prev = val;
|
|||
|
|
}
|
|||
|
|
row[ a.length ] = prev;
|
|||
|
|
}
|
|||
|
|
return row[ a.length ];
|
|||
|
|
};
|
|||
|
|
utils.testinarray = ( array, arrayreferent ) => {
|
|||
|
|
// au moins un element de array existe dans arryreferent
|
|||
|
|
let exist = false;
|
|||
|
|
if( arrayreferent ) {
|
|||
|
|
//console.log('arrrrrrrrrrrrrrr', arrayreferent)
|
|||
|
|
array.forEach( e => {
|
|||
|
|
//console.log(e)
|
|||
|
|
if( arrayreferent.includes( e ) ) exist = true
|
|||
|
|
} )
|
|||
|
|
}
|
|||
|
|
return exist
|
|||
|
|
};
|
|||
|
|
/*
|
|||
|
|
DIRECTORY
|
|||
|
|
*/
|
|||
|
|
const isDirectory = source => fs.lstatSync( source )
|
|||
|
|
.isDirectory();
|
|||
|
|
const getDirectories = source => fs.readdirSync( source )
|
|||
|
|
.map( name => path.join( source, name ) )
|
|||
|
|
.filter( isDirectory );
|
|||
|
|
module.exports = utils;
|