/* eslint-disable no-useless-escape */ const fs = require( 'fs' ); const path = require( 'path' ); const bcrypt = require( 'bcrypt' ); const moment = require( 'moment' ); const config = require( '../config' ); const utils = {}; logger.info( "Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apixtribe project (see /package.json to get email).\n We'll add to the roadmap to add it." ); /** * EMAIL */ /* const validateEmail = email => { const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; return regExp.test(email); }; const validatePassword = pwd => { const regExp = new RegExp( /^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/ ); return regExp.test(pwd); }; const filterInvalidInArray = (array, validate) => array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid /** * POSTAL CODE */ /* const validatePostalCode = postalCode => /(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode); /** * PHONE */ /* const validatePhoneNumber = phoneNumber => /((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber); const correctPhoneNumber = phone => phone[0] === '0' ? '+33' + phone.substr(1) : phone; const checkData = (appProfil, referential, data) => { // @TODO get a referentiel per object then check data validity and allowed access // need to add referentiel manager const invalidefor = []; let updateDatabase = false; Object.keys(data).forEach(field => { switch (field) { case 'token': updateDatabase = true; break; case 'email': if (!validateEmail(data.email)) { invalidefor.push('ERREMAIL:' + field); } else { updateDatabase = true; } break; case 'password': if (!validatePassword(data.password)) { invalidefor.push('ERRPWD:' + field); } else { data.password = bcrypt.hash(data.password, config.saltRounds); updateDatabase = true; } break; } }); return { invalidefor, data, updateDatabase }; }; */ //Permet d'attendre en milliseconde // s'utilise avec async ()=>{ // await sleep(2000) //} utils.sleep = ( ms ) => { return new Promise( resolve => setTimeout( resolve, ms ) ); } utils.generemdp = ( nbpos ) => { const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789"; let mdp = ""; for( var i = 0; i < nbpos; i++ ) { var pos = Math.floor( Math.random() * chaine.length ); mdp += chaine.substring( pos, pos + 1 ); } return mdp; } utils.generecompteur = ( filecpt, typeincrement ) => { let file = `${filecpt}/${typeincrement}.json`; let prefix = ""; if( typeincrement = "ANNEESEMAINE" ) { file = `${filecpt}/${typeincrement}${moment().format('YYYY')}${moment().format('WW')}.json` prefix = `${moment().format('YYYY')}${moment().format('WW')}` } let num = 1; try { num = parseInt( fs.readFileSync( file, 'utf8' ) ) + 1; } catch ( err ) { logger.info( "Nouveau compteur incrementale ", file ) } fs.writeFileSync( file, num, 'utf8' ); return prefix + num } /** * CSV */ utils.json2csv = ( jsondata, options, callback ) => { // uniquement json = [{niv1:val,niv1:[liste of val]}] // logger.info('_________________________'); // logger.info(jsondata) // logger.info('_________________________'); if( jsondata.length == 0 ) { return callback( "Empty json", null ); } if( !options.retln ) options.retln = '\n'; if( !options.sep ) options.sep = ';'; if( !options.arraysplitsep ) options.arraysplitsep = ","; if( !options.replacespecialcarJson2Csv ) { options.replacespecialcarJson2Csv = [] } else { if( typeof options.replacespecialcarJson2Csv == "string" ) { //permet de passer des regex en string options.replacespecialcarJson2Csv = eval( options.replacespecialcarJson2Csv ); } }; let etat = ""; let csv = ''; let entete = ''; let prem = true; for( const j in jsondata ) { // logger.info(jsondata[j]) for( const c in options.champs ) { if( prem ) { entete += options.champs[ c ] + options.sep; } if( jsondata[ j ][ options.champs[ c ] ] ) { if( options.array.indexOf( options.champs[ c ] ) > -1 ) { csv += jsondata[ j ][ options.champs[ c ] ].join( options.arraysplitsep ) + options.sep; } else { let currentValue = ""; if( jsondata[ j ][ options.champs[ c ] ] ) currentValue += jsondata[ j ][ options.champs[ c ] ]; options.replacespecialcarJson2Csv.forEach( re => { //logger.info(currentValue) currentValue = currentValue.replace( re[ 1 ], re[ 0 ] ) } ) csv += currentValue + options.sep; } } else { csv += options.sep; } } csv = csv.substring( 0, csv.length - 1 ) + options.retln; if( prem ) { prem = false; entete = entete.substring( 0, entete.length - 1 ) + options.retln; // logger.info(entete) } } // return entete + csv; if( etat == "" ) { return callback( null, entete + csv ); } else { return callback( etat, null ); } }; /** * Get headers from first line of CSV * @param {array} lines array of string which contains each csv lines * @return {array} string array of headers */ utils.getHeaders = ( lines, sep ) => lines[ 0 ].split( sep ) .map( i => i.replace( /"/g, '' ) ); /** * [csv2json description] * @param {object} csv object of csv file that has been read * @param {object} options object containing csv options, headers, ... {retln:'code de retour de ligne \n ou \n\r', sep:'code to split cells', champs:[ch1,ch2,...] catch only those field, array:[ch1, ] can have more than one field champs with same name then data are push into an array } * @param {Function} callback callback function * @return {callback} - return an error if error, else return json it convert a csv file into a json = [{field:value}] Usage example: fiche.csv2article = (err, fiche) => { if (!err) { logger.info(fiche) } } utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), { retln: "\n", sep: ";", champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"], array: ["TAG", "PHONE_PRO", "EMAIL_PRO"] }, fiche.csv2article) */ utils.replacecarbtweendblquote = ( csv, car, carremplacant ) => { /* return csv text with any car betwenn 2 " by CARSEPARATOR */ let newcsv = ""; let txtencours = ""; let flagouvert = false const sepreg = new RegExp( `${car}`, 'gmi' ) for( let j = 0; j < csv.length; j++ ) { //if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) { if( csv[ j ] == "\"" ) { if( flagouvert ) { // on cherche à ferme une chaine de texte if( csv[ j + 1 ] == "\"" ) { //on a "" consecutif qu'on remplace par "" et on fait j+1 txtencours += "\"\"" j++ } else { // on a bien une fermeture flagouvert = false newcsv += txtencours.replace( sepreg, carremplacant ) txtencours = "\"" } } else { // on ouvre une chaine flagouvert = true //on met le contenu précédent ds newcsv newcsv += txtencours txtencours = "\"" } //} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) { } else if( csv[ j ] !== "\n" ) { txtencours += csv[ j ] // } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) { } else if( csv[ j ] == "\n" ) { if( !flagouvert ) txtencours += "\n" } } return newcsv + txtencours } utils.analysestring = ( string ) => { let buftxt = "" let bufcode = "" let i = 0 let avecRL = false for( let p = 0; p < string.length; p++ ) { if( string[ p ].charCodeAt() == 10 ) { buftxt += "[RL]" avecRL = true } else { buftxt += string[ p ] } bufcode += "-" + string[ p ].charCodeAt(); if( i == 20 ) { if( avecRL ) { logger.info( `${buftxt} - ${bufcode}` ) } else { logger.info( `${buftxt} ---- ${bufcode}` ) } i = 0; buftxt = "" bufcode = "" avecRL = false } i++; } } const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","

 

Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking

",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0, 32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","

 

Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking

",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0, 32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","

PACK JOB SEARCH

",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0, 32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","

PACK JOB SEARCH

",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,` //utils.analysestring(txtstring) //logger.info(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR') // .split("\n")[0].split(",")) utils.csv2json = ( csv, options, callback ) => { // EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT // l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché logger.info( '\n--------------- CSV2JSON ---------------\n' ); // Default CSV options if( !options.retln ) options.retln = '\n'; if( csv.indexOf( '\n\r' ) > -1 ) options.retln = '\n\r'; if( !options.sep ) options.sep = ';'; //gestion d un separateur dans une chaine de texte //const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm'); //csv = csv.replace(regseptext, "CARACSEPAR"); // csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE") csv = utils.replacecarbtweendblquote( csv, options.sep, "CARSEPARATOR" ) if( !options.replacespecialcarCsv2Json ) { options.replacespecialcarCsv2Json = [] } else { if( typeof options.replacespecialcarCsv2Json == "string" ) { //permet de passer des regex en string options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json ); } }; const result = []; const lines = csv.split( options.retln ); const headers = utils.getHeaders( lines, options.sep ); let unknownHeaders = ''; //logger.info('headers', headers) //logger.info('options.champs', options.champs) headers.forEach( header => { // Si un header n'est pas présent dans la liste des champs prédéfinis // on l'ajoute aux champs inconnus if( options.champs.indexOf( header ) === -1 ) { unknownHeaders += `${header}, `; } } ); if( unknownHeaders !== '' ) { const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`; return callback( errorMsg, null ); } lines.forEach( ( line, index ) => { // Skip headers line or empty lines if( index === 0 || line.replace( /\s/g, '' ) .length === 0 ) { return; } // pour debuguer on met origincsv pour voir la ligne d'origine const currentLineData = { 'origincsv': line, 'linenumber': index }; const currentLine = line.split( options.sep ); // Current string in the line for( let j = 0; j < headers.length; j++ ) { // Si la ligne n'est pas vide if( currentLine[ j ] ) { // On clean le champs // ajout eventuel de modification de caracter reservé ; dans les libelléetc... let currentValue = currentLine[ j ].trim() //on transforme le caractere separateur modifié entre double quote currentValue = currentValue.replace( 'CARSEPARATOR', options.sep ); options.replacespecialcarCsv2Json.forEach( re => { currentValue = currentValue.replace( re[ 0 ], re[ 1 ] ) } ) // Si le header est un email if( headers[ j ].includes( 'EMAIL' ) ) { // Supprimer tous les espaces currentValue = currentLine[ j ].replace( /\s/g, '' ); } // on check si le chamos doit être numerique if( options.numericfield.includes( headers[ j ] ) ) { currentValue = currentLine[ j ].replace( /\,/g, '.' ); try { const test = parseFloat( currentValue ); } catch ( er ) { return callback( `${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`, null ); } } if( currentValue ) { // Si le header actuel est de type array // Cela signifie que le header apparaît plusieurs fois dans le CSV // et que les valeurs correspondantes à ce header // doivent être mis dans un array if( options.array && options.array.indexOf( headers[ j ] ) > -1 ) { // Si le tableau pour ce header n'existe pas on le crée if( !currentLineData[ headers[ j ] ] ) { currentLineData[ headers[ j ] ] = []; } if( options.arraysplitsep ) { currentValue.split( options.arraysplitsep ) .forEach( v => { currentLineData[ headers[ j ] ].push( v ); } ) } else { currentLineData[ headers[ j ] ].push( currentValue ); } } else { // Si un header est déjà présent pour la ligne // alors que il n'est pas spécifié comme étant un array // on retourne une erreur if( currentLineData[ headers[ j ] ] ) { const errorMsg = `Le champ ${ headers[j] } est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`; return callback( errorMsg, null ); } currentLineData[ headers[ j ] ] = currentValue; } } } } result.push( currentLineData ); } ); return callback( null, result ); }; /** * [csvparam2json description] * @param {object} csv object of csv file that has been read * @param {object} options object containing csv options, headers, ... {retln:'code de retour de ligne \n ou \n\r', sep:'code to split cells', champs:[ch1,ch2,...] catch only those field, array:[ch1, ] can have more than one field champs with same name then data are push into an array } * @param {Function} callback callback function * @return {callback} - return an error if error, else return json it converts a csv with 3 column col1;col2;col3 in a json in a tree if in col1 we have __ => then it splits a leaf col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored return data = {xxxx:{yyyy:value}} col1 = xxxx; col2 = value; col3 = comment ignored return data = {xxxx:value} Usage example: fiche.csvparam2article = (err, fiche) => { if (!err) { logger.info(fiche) } } utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), { retln: "\n", sep: ";", champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"], array: ["TAG", "PHONE_PRO", "EMAIL_PRO"] }, fiche.csv2article) */ utils.csvparam2json = ( csv, options, callback ) => { logger.info( '\n--------------- CSVPARAM2JSON ---------------\n' ); let etat = ""; const param = {}; if( !options.retln ) { options.retln = '\n'; } if( csv.indexOf( '\n\r' ) > -1 ) { options.retln = '\n\r'; } if( !options.sep ) { options.sep = ';'; } if( !options.seplevel ) { options.seplevel = "__"; } if( !options.replacespecialcarCsv2Json ) { options.replacespecialcarCsv2Json = [] } else { if( typeof options.replacespecialcarCsv2Json == "string" ) { //permet de passer des regex en string options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json ); } }; const lines = csv.split( options.retln ); for( let i = 0; i < lines.length; i++ ) { const infol = lines[ i ].split( options.sep ) //logger.info(infol) if( infol[ 0 ].length > 4 && infol.length < 2 ) { // si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`; continue; } // On ajoute ici la gestion de tous les caracteres spéciaux // reservées pour le csv ; ' etc..' if( infol[ 1 ] && infol[ 1 ] + "" == infol[ 1 ] ) { options.replacespecialcarCsv2Json.forEach( re => { //logger.info("gggggggggggggggggggg", infol[1]) infol[ 1 ] = infol[ 1 ].replace( re[ 0 ], re[ 1 ] ); } ) // logger.info(infol[1]) infol[ 1 ] = infol[ 1 ].replace( /'|’/g, "\"" ); //logger.info(infol[1]) if( infol[ 1 ].toLowerCase() === 'true' ) { infol[ 1 ] = true; } else if( infol[ 1 ].toLowerCase() === 'false' ) { infol[ 1 ] = false; } } logger.info( infol[ 1 ] ) //supprime des lignes vides if( infol[ 0 ] == '' ) continue; if( infol[ 0 ].indexOf( options.seplevel ) == -1 ) { param[ infol[ 0 ] ] = infol[ 1 ] continue; } else { const arbre = infol[ 0 ].split( options.seplevel ) switch ( arbre.length ) { case 1: param[ arbre[ 0 ] ] = infol[ 1 ]; break; case 2: if( arbre[ 1 ] != "ARRAY" ) { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {}; param[ arbre[ 0 ] ][ arbre[ 1 ] ] = infol[ 1 ]; } else { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = []; //logger.info('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"')) eval( "result=" + infol[ 1 ] ) //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[ arbre[ 0 ] ].push( result ) } break; case 3: if( arbre[ 2 ] != "ARRAY" ) { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {}; param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = infol[ 1 ]; } else { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = []; //eval("result = \"test\""); //logger.info(result); eval( "result=" + infol[ 1 ] ); //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[ arbre[ 0 ] ][ arbre[ 1 ] ].push( result ) } break; case 4: if( arbre[ 3 ] != "ARRAY" ) { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = {}; param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ][ arbre[ 3 ] ] = infol[ 1 ]; } else { if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {}; if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = []; eval( "result=" + infol[ 1 ] ) //.substring(1, infol[1].length - 1).replace(/""/g, '"')) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ].push( result ) break; } default: break; } } } // JSON.parse(JSON.stringify(param)) logger.info( 'kkkkkkkkkkkkkkkkkk', param[ 'catalogue' ][ 'filtrecatalog' ][ 'searchengine' ] ) if( etat == "" ) { return callback( null, JSON.parse( JSON.stringify( param ) ) ); } else { return callback( etat, null ); } } utils.levenshtein = ( a, b ) => { if( a.length === 0 ) return b.length; if( b.length === 0 ) return a.length; let tmp, i, j, prev, val, row; // swap to save some memory O(min(a,b)) instead of O(a) if( a.length > b.length ) { tmp = a; a = b; b = tmp; } row = Array( a.length + 1 ); // init the row for( i = 0; i <= a.length; i++ ) { row[ i ] = i; } // fill in the rest for( i = 1; i <= b.length; i++ ) { prev = i; for( j = 1; j <= a.length; j++ ) { if( b[ i - 1 ] === a[ j - 1 ] ) { val = row[ j - 1 ]; // match } else { val = Math.min( row[ j - 1 ] + 1, // substitution Math.min( prev + 1, // insertion row[ j ] + 1 ) ); // deletion } row[ j - 1 ] = prev; prev = val; } row[ a.length ] = prev; } return row[ a.length ]; }; utils.testinarray = ( array, arrayreferent ) => { // au moins un element de array existe dans arryreferent let exist = false; if( arrayreferent ) { //logger.info('arrrrrrrrrrrrrrr', arrayreferent) array.forEach( e => { //logger.info(e) if( arrayreferent.includes( e ) ) exist = true } ) } return exist }; /* DIRECTORY */ const isDirectory = source => fs.lstatSync( source ) .isDirectory(); const getDirectories = source => fs.readdirSync( source ) .map( name => path.join( source, name ) ) .filter( isDirectory ); module.exports = utils;