first new commit
This commit is contained in:
185
nationchains/socialworld/contracts/checkdata.js
Executable file
185
nationchains/socialworld/contracts/checkdata.js
Executable file
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
This module have to be independant of any external package
|
||||
it is shared between back and front and is usefull
|
||||
to apply common check in front before sending it in back
|
||||
can be include in project with
|
||||
<script src="https://apiback.maildigit.fr/js/checkdata.js"></script>
|
||||
or with const checkdata = require('../public/js/checkdata.js')
|
||||
|
||||
*/
|
||||
|
||||
// --##
|
||||
|
||||
const checkdata = {};
|
||||
// each checkdata.test. return true or false
|
||||
checkdata.test = {};
|
||||
|
||||
checkdata.test.emailadress = ( ctx, email ) => {
|
||||
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
|
||||
return regExp.test( email );
|
||||
};
|
||||
/*
|
||||
* @emaillist = "email1,email2, email3"
|
||||
* it check if each eamil separate by , are correct
|
||||
*/
|
||||
checkdata.test.emailadresslist = ( ctx, emaillist ) => {
|
||||
//console.log(emaillist.split(','))
|
||||
if( emaillist.length > 0 ) {
|
||||
const emails = emaillist.split( ',' );
|
||||
for( var i in emails ) {
|
||||
//console.log(emails[i])
|
||||
if( !checkdata.test.emailadress( "", emails[ i ].trim() ) ) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
};
|
||||
return true;
|
||||
};
|
||||
|
||||
checkdata.test.password = ( ctx, pwd ) => {
|
||||
const regExp = new RegExp(
|
||||
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
|
||||
);
|
||||
return regExp.test( pwd );
|
||||
};
|
||||
checkdata.test.required = ( ctx, val ) =>
|
||||
( val != null && val != 'undefined' && val.length > 0 ) || ( !!val && val.constructor === Array && val.length > 0 ) || ( !!val && val.constructor === Object && Object.keys( val )
|
||||
.length > 0 );
|
||||
|
||||
checkdata.test.isNumber = ( ctx, n ) => typeof n === 'number';
|
||||
checkdata.test.isInt = ( ctx, n ) => n != '' && !isNaN( n ) && Math.round( n ) == n;
|
||||
checkdata.test.isFloat = ( ctx, n ) => n != '' && !isNaN( n ) && Math.round( n ) != n;
|
||||
checkdata.test.unique = ( ctx, val ) => {
|
||||
if( ctx.list[ ctx.currentfield ] ) {
|
||||
return !ctx.list[ ctx.currentfield ].includes( val );
|
||||
} else {
|
||||
console.log( 'ERR no list for field:' + ctx.currentfield );
|
||||
return false;
|
||||
}
|
||||
};
|
||||
checkdata.test.isDateDay = ( ctx, dateDay ) => true;
|
||||
/* checkdata.test.filterInvalidInArray = (array, validate) =>
|
||||
array ? array.filter(el => !validate(el)) : true;
|
||||
// return true when every elements is valid
|
||||
*/
|
||||
|
||||
checkdata.test.postalCode = ( ctx, postalCode ) => {
|
||||
if( postalCode.length == 0 ) return true;
|
||||
const regExp = new RegExp( /(^\d{5}$)|(^\d{5}-\d{4}$)/ );
|
||||
return regExp.test( postalCode );
|
||||
};
|
||||
/**
|
||||
* PHONE
|
||||
*/
|
||||
checkdata.test.phoneNumber = ( ctx, phoneNumber ) => {
|
||||
if( phoneNumber.length == 0 ) return true;
|
||||
phoneNumber = phoneNumber.trim()
|
||||
.replace( /[- .]/g, '' )
|
||||
//french number
|
||||
const regExpfr = new RegExp( /^0[1-9][0-9]{9}$/ );
|
||||
const regExpInternational = new RegExp( /^\+*(\d{3})*[0-9,\-]{8,}/ );
|
||||
return regExpfr.test( phoneNumber ) || regExpInternational.test( phoneNumber );
|
||||
};
|
||||
/*
|
||||
* @phonelist = "phone1,phone2,phone3"
|
||||
* it check if each phone separate by , are correct
|
||||
*/
|
||||
checkdata.test.phoneNumberlist = ( ctx, phonelist ) => {
|
||||
//console.log(emaillist.split(','))
|
||||
if( phonelist.length > 0 ) {
|
||||
const phones = phonelist.split( ',' );
|
||||
for( var i in phones ) {
|
||||
//console.log(emails[i])
|
||||
if( !checkdata.test.phoneNumber( "", phones[ i ].trim() ) ) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
};
|
||||
return true;
|
||||
};
|
||||
|
||||
// checkdata.normalize take a correct data then reformat it to harmonise it
|
||||
checkdata.normalize = {};
|
||||
checkdata.normalize.phoneNumber = ( ctx, phone ) => {
|
||||
phone = phone.trim()
|
||||
.replace( /[- .]/g, '' );
|
||||
if( checkdata.test.phoneNumber( '', phone ) && phone.length == 10 && phone[ 0 ] == "0" ) {
|
||||
phone = '+33 ' + phone.substring( 1 );
|
||||
}
|
||||
return phone;
|
||||
}
|
||||
checkdata.normalize.upperCase = ( ctx, txt ) => txt.toUpperCase();
|
||||
checkdata.normalize.lowerCase = ( ctx, txt ) => txt.toLowerCase();
|
||||
// fixe 10 position et complete par des 0 devant
|
||||
checkdata.normalize.zfill10 = ( ctx, num ) => {
|
||||
let s = num + '';
|
||||
while( s.length < 10 ) s = '0' + s;
|
||||
return s;
|
||||
};
|
||||
/*let tt = "+33 1 02.03 04 05";
|
||||
console.log(checkdata.test.phoneNumber('', tt))
|
||||
console.log(checkdata.normalize.phoneNumber('', tt))
|
||||
*/
|
||||
checkdata.evaluate = ( contexte, referential, data ) => {
|
||||
/*
|
||||
* contexte object {} with full info for evaluation
|
||||
* file referential path to get object to apply
|
||||
* data related to object
|
||||
- return {validefor =[keyword of error] if empty no error,
|
||||
clean data eventually reformated
|
||||
updateDatabase}
|
||||
*/
|
||||
console.log( 'contexte', contexte );
|
||||
console.log( 'referentiel', referential );
|
||||
console.log( 'data', data );
|
||||
const invalidefor = [];
|
||||
const objectdef = {};
|
||||
const listfield = referential.map( ch => {
|
||||
objectdef[ ch.idfield ] = ch;
|
||||
return ch.idfield;
|
||||
} );
|
||||
|
||||
Object.keys( data )
|
||||
.forEach( field => {
|
||||
if( !listfield.includes( field ) ) {
|
||||
// some data can be inside an object with no control at all
|
||||
// they are used for process only
|
||||
// i leave it in case it will become a non sens
|
||||
// invalidefor.push('ERRFIELD unknown of referentials ' + field);
|
||||
} else {
|
||||
if( objectdef[ field ].check ) {
|
||||
// check data with rule list in check
|
||||
objectdef[ field ].check.forEach( ctrl => {
|
||||
console.log( 'ctrl', ctrl );
|
||||
contexte.currentfield = field;
|
||||
if( !checkdata.test[ ctrl ] ) {
|
||||
invalidefor.push( 'ERR check function does not exist :' + ctrl + '___' + field )
|
||||
} else {
|
||||
if( !checkdata.test[ ctrl ]( contexte, data[ field ] ) )
|
||||
invalidefor.push( 'ERR' + ctrl + '___' + field );
|
||||
}
|
||||
} );
|
||||
}
|
||||
|
||||
if( objectdef[ field ].nouserupdate ) {
|
||||
// check if user can modify this information
|
||||
console.log(
|
||||
'evaluation :' + field + ' -- ' + objectdef[ field ].nouserupdate,
|
||||
eval( objectdef[ field ].nouserupdate )
|
||||
);
|
||||
const evalright = eval( objectdef[ field ].nouserupdate );
|
||||
objectdef[ field ].nouserupdate = evalright;
|
||||
}
|
||||
}
|
||||
} );
|
||||
console.log( {
|
||||
invalidefor,
|
||||
data
|
||||
} );
|
||||
return {
|
||||
invalidefor,
|
||||
data
|
||||
};
|
||||
};
|
||||
|
||||
if( typeof module !== 'undefined' ) module.exports = checkdata;
|
||||
604
nationchains/socialworld/contracts/toolsbox.js
Executable file
604
nationchains/socialworld/contracts/toolsbox.js
Executable file
@@ -0,0 +1,604 @@
|
||||
/* eslint-disable no-useless-escape */
|
||||
const fs = require( 'fs' );
|
||||
const path = require( 'path' );
|
||||
const bcrypt = require( 'bcrypt' );
|
||||
const moment = require( 'moment' );
|
||||
const config = require( '../config' );
|
||||
const utils = {};
|
||||
|
||||
console.log( "Check in /utils/index.js to find usefull function for your dev.\n Feel free to send suggestion, code to maintainer of apixtribe project (see /package.json to get email).\n We'll add to the roadmap to add it." );
|
||||
|
||||
/**
|
||||
* EMAIL
|
||||
*/
|
||||
/* const validateEmail = email => {
|
||||
const regExp = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
|
||||
return regExp.test(email);
|
||||
};
|
||||
|
||||
const validatePassword = pwd => {
|
||||
const regExp = new RegExp(
|
||||
/^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[$@$!%*?&.])[A-Za-z\d$@$!%*?&.{}:|\s]{8,}/
|
||||
);
|
||||
return regExp.test(pwd);
|
||||
};
|
||||
|
||||
const filterInvalidInArray = (array, validate) =>
|
||||
array ? array.filter(el => !validate(el)) : undefined; // return undefined when every elements is valid
|
||||
|
||||
/**
|
||||
* POSTAL CODE
|
||||
*/
|
||||
/*
|
||||
const validatePostalCode = postalCode =>
|
||||
/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(postalCode);
|
||||
|
||||
/**
|
||||
* PHONE
|
||||
*/
|
||||
/* const validatePhoneNumber = phoneNumber =>
|
||||
/((^0[1-9]|\+[0-9]{3})([-. ]?[0-9]{2}){4}$)/.test(phoneNumber);
|
||||
|
||||
const correctPhoneNumber = phone =>
|
||||
phone[0] === '0' ? '+33' + phone.substr(1) : phone;
|
||||
|
||||
const checkData = (appProfil, referential, data) => {
|
||||
// @TODO get a referentiel per object then check data validity and allowed access
|
||||
// need to add referentiel manager
|
||||
const invalidefor = [];
|
||||
let updateDatabase = false;
|
||||
Object.keys(data).forEach(field => {
|
||||
switch (field) {
|
||||
case 'token':
|
||||
updateDatabase = true;
|
||||
break;
|
||||
case 'email':
|
||||
if (!validateEmail(data.email)) {
|
||||
invalidefor.push('ERREMAIL:' + field);
|
||||
} else {
|
||||
updateDatabase = true;
|
||||
}
|
||||
break;
|
||||
case 'password':
|
||||
if (!validatePassword(data.password)) {
|
||||
invalidefor.push('ERRPWD:' + field);
|
||||
} else {
|
||||
data.password = bcrypt.hash(data.password, config.saltRounds);
|
||||
updateDatabase = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
return { invalidefor, data, updateDatabase };
|
||||
};
|
||||
*/
|
||||
//Permet d'attendre en milliseconde
|
||||
// s'utilise avec async ()=>{
|
||||
// await sleep(2000)
|
||||
//}
|
||||
utils.sleep = ( ms ) => {
|
||||
return new Promise( resolve => setTimeout( resolve, ms ) );
|
||||
}
|
||||
utils.generemdp = ( nbpos ) => {
|
||||
const chaine = "ABCDEFGHIJKLMNPQRSTUVWZY123456789";
|
||||
let mdp = "";
|
||||
for( var i = 0; i < nbpos; i++ ) {
|
||||
var pos = Math.floor( Math.random() * chaine.length );
|
||||
mdp += chaine.substring( pos, pos + 1 );
|
||||
}
|
||||
return mdp;
|
||||
}
|
||||
utils.generecompteur = ( filecpt, typeincrement ) => {
|
||||
let file = `${filecpt}/${typeincrement}.json`;
|
||||
let prefix = "";
|
||||
if( typeincrement = "ANNEESEMAINE" ) {
|
||||
file = `${filecpt}/${typeincrement}${moment().format('YYYY')}${moment().format('WW')}.json`
|
||||
prefix = `${moment().format('YYYY')}${moment().format('WW')}`
|
||||
}
|
||||
let num = 1;
|
||||
try {
|
||||
num = parseInt( fs.readFileSync( file, 'utf8' ) ) + 1;
|
||||
} catch ( err ) {
|
||||
console.log( "Nouveau compteur incrementale ", file )
|
||||
}
|
||||
fs.writeFileSync( file, num, 'utf8' );
|
||||
return prefix + num
|
||||
}
|
||||
/**
|
||||
* CSV
|
||||
*/
|
||||
utils.json2csv = ( jsondata, options, callback ) => {
|
||||
// uniquement json = [{niv1:val,niv1:[liste of val]}]
|
||||
// console.log('_________________________');
|
||||
// console.log(jsondata)
|
||||
// console.log('_________________________');
|
||||
if( jsondata.length == 0 ) {
|
||||
return callback( "Empty json", null );
|
||||
}
|
||||
if( !options.retln ) options.retln = '\n';
|
||||
if( !options.sep ) options.sep = ';';
|
||||
if( !options.arraysplitsep ) options.arraysplitsep = ",";
|
||||
if( !options.replacespecialcarJson2Csv ) {
|
||||
options.replacespecialcarJson2Csv = []
|
||||
} else {
|
||||
if( typeof options.replacespecialcarJson2Csv == "string" ) {
|
||||
//permet de passer des regex en string
|
||||
options.replacespecialcarJson2Csv = eval( options.replacespecialcarJson2Csv );
|
||||
}
|
||||
};
|
||||
let etat = "";
|
||||
let csv = '';
|
||||
let entete = '';
|
||||
let prem = true;
|
||||
for( const j in jsondata ) {
|
||||
// console.log(jsondata[j])
|
||||
for( const c in options.champs ) {
|
||||
if( prem ) {
|
||||
entete += options.champs[ c ] + options.sep;
|
||||
}
|
||||
if( jsondata[ j ][ options.champs[ c ] ] ) {
|
||||
if( options.array.indexOf( options.champs[ c ] ) > -1 ) {
|
||||
csv += jsondata[ j ][ options.champs[ c ] ].join( options.arraysplitsep ) + options.sep;
|
||||
} else {
|
||||
let currentValue = "";
|
||||
if( jsondata[ j ][ options.champs[ c ] ] ) currentValue += jsondata[ j ][ options.champs[ c ] ];
|
||||
options.replacespecialcarJson2Csv.forEach( re => {
|
||||
//console.log(currentValue)
|
||||
currentValue = currentValue.replace( re[ 1 ], re[ 0 ] )
|
||||
} )
|
||||
csv += currentValue + options.sep;
|
||||
}
|
||||
} else {
|
||||
csv += options.sep;
|
||||
}
|
||||
}
|
||||
csv = csv.substring( 0, csv.length - 1 ) + options.retln;
|
||||
if( prem ) {
|
||||
prem = false;
|
||||
entete = entete.substring( 0, entete.length - 1 ) + options.retln;
|
||||
// console.log(entete)
|
||||
}
|
||||
}
|
||||
// return entete + csv;
|
||||
if( etat == "" ) {
|
||||
return callback( null, entete + csv );
|
||||
} else {
|
||||
return callback( etat, null );
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Get headers from first line of CSV
|
||||
* @param {array} lines array of string which contains each csv lines
|
||||
* @return {array} string array of headers
|
||||
*/
|
||||
utils.getHeaders = ( lines, sep ) => lines[ 0 ].split( sep )
|
||||
.map( i => i.replace( /"/g, '' ) );
|
||||
/**
|
||||
* [csv2json description]
|
||||
* @param {object} csv object of csv file that has been read
|
||||
* @param {object} options object containing csv options, headers, ...
|
||||
{retln:'code de retour de ligne \n ou \n\r',
|
||||
sep:'code to split cells',
|
||||
champs:[ch1,ch2,...] catch only those field,
|
||||
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
||||
* @param {Function} callback callback function
|
||||
* @return {callback} - return an error if error, else return json
|
||||
it convert a csv file into a json = [{field:value}]
|
||||
|
||||
Usage example:
|
||||
fiche.csv2article = (err, fiche) => {
|
||||
if (!err) {
|
||||
console.log(fiche)
|
||||
}
|
||||
}
|
||||
utils.csv2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
||||
retln: "\n",
|
||||
sep: ";",
|
||||
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
||||
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
||||
}, fiche.csv2article)
|
||||
|
||||
*/
|
||||
utils.replacecarbtweendblquote = ( csv, car, carremplacant ) => {
|
||||
/*
|
||||
return csv text with any car betwenn 2 " by CARSEPARATOR
|
||||
*/
|
||||
let newcsv = "";
|
||||
let txtencours = "";
|
||||
let flagouvert = false
|
||||
const sepreg = new RegExp( `${car}`, 'gmi' )
|
||||
for( let j = 0; j < csv.length; j++ ) {
|
||||
//if((csv[j] == "\"" && csv[j + 1] && csv[j + 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 1] != "\"") || (csv[j] == "\"" && csv[j - 1] && csv[j - 2] && csv[j - 1] != "\"" && csv[j - 2] != "\"")) {
|
||||
if( csv[ j ] == "\"" ) {
|
||||
if( flagouvert ) {
|
||||
// on cherche à ferme une chaine de texte
|
||||
if( csv[ j + 1 ] == "\"" ) {
|
||||
//on a "" consecutif qu'on remplace par "" et on fait j+1
|
||||
txtencours += "\"\""
|
||||
j++
|
||||
} else {
|
||||
// on a bien une fermeture
|
||||
flagouvert = false
|
||||
newcsv += txtencours.replace( sepreg, carremplacant )
|
||||
txtencours = "\""
|
||||
}
|
||||
} else {
|
||||
// on ouvre une chaine
|
||||
flagouvert = true
|
||||
//on met le contenu précédent ds newcsv
|
||||
newcsv += txtencours
|
||||
txtencours = "\""
|
||||
}
|
||||
//} else if((csv[j] !== "\n") && (csv[j + 1] && csv[j] + csv[j + 1] !== "\n\r")) {
|
||||
} else if( csv[ j ] !== "\n" ) {
|
||||
txtencours += csv[ j ]
|
||||
// } else if((csv[j] == "\n") || (csv[j + 1] && csv[j] + csv[j + 1] == "\n\r")) {
|
||||
} else if( csv[ j ] == "\n" ) {
|
||||
if( !flagouvert ) txtencours += "\n"
|
||||
}
|
||||
}
|
||||
return newcsv + txtencours
|
||||
}
|
||||
utils.analysestring = ( string ) => {
|
||||
let buftxt = ""
|
||||
let bufcode = ""
|
||||
let i = 0
|
||||
let avecRL = false
|
||||
for( let p = 0; p < string.length; p++ ) {
|
||||
if( string[ p ].charCodeAt() == 10 ) {
|
||||
buftxt += "[RL]"
|
||||
avecRL = true
|
||||
} else {
|
||||
buftxt += string[ p ]
|
||||
}
|
||||
bufcode += "-" + string[ p ].charCodeAt();
|
||||
if( i == 20 ) {
|
||||
if( avecRL ) {
|
||||
console.log( `${buftxt} - ${bufcode}` )
|
||||
} else {
|
||||
console.log( `${buftxt} ---- ${bufcode}` )
|
||||
}
|
||||
i = 0;
|
||||
buftxt = ""
|
||||
bufcode = ""
|
||||
avecRL = false
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
const txtstring = `32932,BK_F2F_B_COM_10x1H-09,"My Communication Workshop ""Session N°9 - 1H""","<p> </p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32932,1101,,"BUSINESS KEYS",0,
|
||||
32933,BK_F2F_B_COM_10x1H-10,"My Communication Workshop Session N°10 - 1H","<p> </p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<p>Learner who needs to develop their ability to communicate effectively at work, both in writing and speaking</p>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>",,english,2,0,,2,0,classroom,"0000-00-00 00:00:00","0000-00-00 00:00:00",0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32933,1101,,"BUSINESS KEYS",0,
|
||||
32934,BK_F2F_B_JOB_10x1H-01,"My Job Search Workshop Session N°1 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32934,1108,,,0,
|
||||
32935,BK_F2F_B_JOB_10x1H-02,"My Job Search Workshop Session N°2 - 1H","<p>PACK JOB SEARCH</p>",,english,2,0,,2,0,classroom,,,0000-00-00,0000-00-00,https://www.yesnyoulearning.com/lms/index.php?r=player&course_id=32935,1108,,,0,`
|
||||
//utils.analysestring(txtstring)
|
||||
//console.log(utils.replacecarbtweendblquote(txtstring, ",", 'CARSEPARATOR')
|
||||
// .split("\n")[0].split(","))
|
||||
utils.csv2json = ( csv, options, callback ) => {
|
||||
// EN CAS DE PB AVEC UN FICHIER EXCEL RECALCITRANT
|
||||
// l'ouvrir dans calc linux et sauvegarder csv utf8, ; , " enregistrer le contenu de la cellule comme affiché
|
||||
console.log( '\n--------------- CSV2JSON ---------------\n' );
|
||||
// Default CSV options
|
||||
if( !options.retln ) options.retln = '\n';
|
||||
if( csv.indexOf( '\n\r' ) > -1 ) options.retln = '\n\r';
|
||||
if( !options.sep ) options.sep = ';';
|
||||
//gestion d un separateur dans une chaine de texte
|
||||
//const regseptext = new RegExp(`${options.sep}(?!(?:[^"]*"[^"]*")*[^"]*$)`, 'gm');
|
||||
//csv = csv.replace(regseptext, "CARACSEPAR");
|
||||
// csv = utils.replacecarbtweendblquote(csv, options.retln, "RETLIGNE")
|
||||
csv = utils.replacecarbtweendblquote( csv, options.sep, "CARSEPARATOR" )
|
||||
if( !options.replacespecialcarCsv2Json ) {
|
||||
options.replacespecialcarCsv2Json = []
|
||||
} else {
|
||||
if( typeof options.replacespecialcarCsv2Json == "string" ) {
|
||||
//permet de passer des regex en string
|
||||
options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json );
|
||||
}
|
||||
};
|
||||
const result = [];
|
||||
const lines = csv.split( options.retln );
|
||||
const headers = utils.getHeaders( lines, options.sep );
|
||||
let unknownHeaders = '';
|
||||
//console.log('headers', headers)
|
||||
//console.log('options.champs', options.champs)
|
||||
headers.forEach( header => {
|
||||
// Si un header n'est pas présent dans la liste des champs prédéfinis
|
||||
// on l'ajoute aux champs inconnus
|
||||
if( options.champs.indexOf( header ) === -1 ) {
|
||||
unknownHeaders += `${header}, `;
|
||||
}
|
||||
} );
|
||||
if( unknownHeaders !== '' ) {
|
||||
const errorMsg = `CSV2JSON() - Champs inconnus : ${unknownHeaders}`;
|
||||
return callback( errorMsg, null );
|
||||
}
|
||||
lines.forEach( ( line, index ) => {
|
||||
// Skip headers line or empty lines
|
||||
if( index === 0 || line.replace( /\s/g, '' )
|
||||
.length === 0 ) {
|
||||
return;
|
||||
}
|
||||
// pour debuguer on met origincsv pour voir la ligne d'origine
|
||||
const currentLineData = { 'origincsv': line, 'linenumber': index };
|
||||
const currentLine = line.split( options.sep ); // Current string in the line
|
||||
for( let j = 0; j < headers.length; j++ ) {
|
||||
// Si la ligne n'est pas vide
|
||||
if( currentLine[ j ] ) {
|
||||
// On clean le champs
|
||||
// ajout eventuel de modification de caracter reservé ; dans les libelléetc...
|
||||
let currentValue = currentLine[ j ].trim()
|
||||
//on transforme le caractere separateur modifié entre double quote
|
||||
currentValue = currentValue.replace( 'CARSEPARATOR', options.sep );
|
||||
options.replacespecialcarCsv2Json.forEach( re => {
|
||||
currentValue = currentValue.replace( re[ 0 ], re[ 1 ] )
|
||||
} )
|
||||
// Si le header est un email
|
||||
if( headers[ j ].includes( 'EMAIL' ) ) {
|
||||
// Supprimer tous les espaces
|
||||
currentValue = currentLine[ j ].replace( /\s/g, '' );
|
||||
}
|
||||
// on check si le chamos doit être numerique
|
||||
if( options.numericfield.includes( headers[ j ] ) ) {
|
||||
currentValue = currentLine[ j ].replace( /\,/g, '.' );
|
||||
try {
|
||||
const test = parseFloat( currentValue );
|
||||
} catch ( er ) {
|
||||
return callback( `${headers[j]} contiens la valeur -${currentValue}- et devrait être numerique`, null );
|
||||
}
|
||||
}
|
||||
if( currentValue ) {
|
||||
// Si le header actuel est de type array
|
||||
// Cela signifie que le header apparaît plusieurs fois dans le CSV
|
||||
// et que les valeurs correspondantes à ce header
|
||||
// doivent être mis dans un array
|
||||
if( options.array && options.array.indexOf( headers[ j ] ) > -1 ) {
|
||||
// Si le tableau pour ce header n'existe pas on le crée
|
||||
if( !currentLineData[ headers[ j ] ] ) {
|
||||
currentLineData[ headers[ j ] ] = [];
|
||||
}
|
||||
if( options.arraysplitsep ) {
|
||||
currentValue.split( options.arraysplitsep )
|
||||
.forEach( v => {
|
||||
currentLineData[ headers[ j ] ].push( v );
|
||||
} )
|
||||
} else {
|
||||
currentLineData[ headers[ j ] ].push( currentValue );
|
||||
}
|
||||
} else {
|
||||
// Si un header est déjà présent pour la ligne
|
||||
// alors que il n'est pas spécifié comme étant un array
|
||||
// on retourne une erreur
|
||||
if( currentLineData[ headers[ j ] ] ) {
|
||||
const errorMsg = `Le champ ${
|
||||
headers[j]
|
||||
} est présent plusieurs fois alors qu'il n'est pas spécifié comme étant un array !`;
|
||||
return callback( errorMsg, null );
|
||||
}
|
||||
currentLineData[ headers[ j ] ] = currentValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.push( currentLineData );
|
||||
} );
|
||||
return callback( null, result );
|
||||
};
|
||||
/**
|
||||
* [csvparam2json description]
|
||||
* @param {object} csv object of csv file that has been read
|
||||
* @param {object} options object containing csv options, headers, ...
|
||||
{retln:'code de retour de ligne \n ou \n\r',
|
||||
sep:'code to split cells',
|
||||
champs:[ch1,ch2,...] catch only those field,
|
||||
array:[ch1, ] can have more than one field champs with same name then data are push into an array }
|
||||
* @param {Function} callback callback function
|
||||
* @return {callback} - return an error if error, else return json
|
||||
it converts a csv with 3 column col1;col2;col3 in a json in a tree
|
||||
if in col1 we have __ => then it splits a leaf
|
||||
col1 = xxxx__yyyy ; col2 = value ; col3 = comment that is ignored
|
||||
return data = {xxxx:{yyyy:value}}
|
||||
col1 = xxxx; col2 = value; col3 = comment ignored
|
||||
return data = {xxxx:value}
|
||||
|
||||
Usage example:
|
||||
fiche.csvparam2article = (err, fiche) => {
|
||||
if (!err) {
|
||||
console.log(fiche)
|
||||
}
|
||||
}
|
||||
utils.csvparam2json(fs.readFileSync('./devdata/tribee/aubergenville/infoexterne/localbusiness.csv', 'utf-8'), {
|
||||
retln: "\n",
|
||||
sep: ";",
|
||||
champs: ["NOM", "OBJET", "ADRESSE_PRO", "CP_PRO", "VILLE_PRO", "ZONE", "PHONE_PRO", "HORAIRESDESC", "HORAIREDATA", "URL", "FACEBOOK", "INSTA", "EMAIL_PRO", "IMG", "TAG"],
|
||||
array: ["TAG", "PHONE_PRO", "EMAIL_PRO"]
|
||||
}, fiche.csv2article)
|
||||
|
||||
*/
|
||||
utils.csvparam2json = ( csv, options, callback ) => {
|
||||
console.log( '\n--------------- CSVPARAM2JSON ---------------\n' );
|
||||
let etat = "";
|
||||
const param = {};
|
||||
if( !options.retln ) {
|
||||
options.retln = '\n';
|
||||
}
|
||||
if( csv.indexOf( '\n\r' ) > -1 ) {
|
||||
options.retln = '\n\r';
|
||||
}
|
||||
if( !options.sep ) {
|
||||
options.sep = ';';
|
||||
}
|
||||
if( !options.seplevel ) {
|
||||
options.seplevel = "__";
|
||||
}
|
||||
if( !options.replacespecialcarCsv2Json ) {
|
||||
options.replacespecialcarCsv2Json = []
|
||||
} else {
|
||||
if( typeof options.replacespecialcarCsv2Json == "string" ) {
|
||||
//permet de passer des regex en string
|
||||
options.replacespecialcarCsv2Json = eval( options.replacespecialcarCsv2Json );
|
||||
}
|
||||
};
|
||||
const lines = csv.split( options.retln );
|
||||
for( let i = 0; i < lines.length; i++ ) {
|
||||
const infol = lines[ i ].split( options.sep )
|
||||
//console.log(infol)
|
||||
if( infol[ 0 ].length > 4 && infol.length < 2 ) {
|
||||
// si le 1er element à plus de 4 caractere et s'il y a moins de 3 colonnes c'est qu'il y a un pb
|
||||
etat += `Erreur sur ${lines[i]} moins de 3 column separé par ${options.sep}`;
|
||||
continue;
|
||||
}
|
||||
// On ajoute ici la gestion de tous les caracteres spéciaux
|
||||
// reservées pour le csv ; ' etc..'
|
||||
if( infol[ 1 ] && infol[ 1 ] + "" == infol[ 1 ] ) {
|
||||
options.replacespecialcarCsv2Json.forEach( re => {
|
||||
//console.log("gggggggggggggggggggg", infol[1])
|
||||
infol[ 1 ] = infol[ 1 ].replace( re[ 0 ], re[ 1 ] );
|
||||
} )
|
||||
// console.log(infol[1])
|
||||
infol[ 1 ] = infol[ 1 ].replace( /'|’/g, "\"" );
|
||||
//console.log(infol[1])
|
||||
if( infol[ 1 ].toLowerCase() === 'true' ) {
|
||||
infol[ 1 ] = true;
|
||||
} else if( infol[ 1 ].toLowerCase() === 'false' ) {
|
||||
infol[ 1 ] = false;
|
||||
}
|
||||
}
|
||||
console.log( infol[ 1 ] )
|
||||
//supprime des lignes vides
|
||||
if( infol[ 0 ] == '' ) continue;
|
||||
if( infol[ 0 ].indexOf( options.seplevel ) == -1 ) {
|
||||
param[ infol[ 0 ] ] = infol[ 1 ]
|
||||
continue;
|
||||
} else {
|
||||
const arbre = infol[ 0 ].split( options.seplevel )
|
||||
switch ( arbre.length ) {
|
||||
case 1:
|
||||
param[ arbre[ 0 ] ] = infol[ 1 ];
|
||||
break;
|
||||
case 2:
|
||||
if( arbre[ 1 ] != "ARRAY" ) {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
||||
param[ arbre[ 0 ] ][ arbre[ 1 ] ] = infol[ 1 ];
|
||||
} else {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = [];
|
||||
//console.log('aff', infol[1].substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
||||
eval( "result=" + infol[ 1 ] )
|
||||
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
||||
param[ arbre[ 0 ] ].push( result )
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
if( arbre[ 2 ] != "ARRAY" ) {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
||||
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = infol[ 1 ];
|
||||
} else {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = [];
|
||||
//eval("result = \"test\"");
|
||||
//console.log(result);
|
||||
eval( "result=" + infol[ 1 ] );
|
||||
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
||||
param[ arbre[ 0 ] ][ arbre[ 1 ] ].push( result )
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
if( arbre[ 3 ] != "ARRAY" ) {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = {};
|
||||
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ][ arbre[ 3 ] ] = infol[ 1 ];
|
||||
} else {
|
||||
if( !param[ arbre[ 0 ] ] ) param[ arbre[ 0 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ] = {};
|
||||
if( !param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] ) param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ] = [];
|
||||
eval( "result=" + infol[ 1 ] )
|
||||
//.substring(1, infol[1].length - 1).replace(/""/g, '"'))
|
||||
param[ arbre[ 0 ] ][ arbre[ 1 ] ][ arbre[ 2 ] ].push( result )
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// JSON.parse(JSON.stringify(param))
|
||||
console.log( 'kkkkkkkkkkkkkkkkkk', param[ 'catalogue' ][ 'filtrecatalog' ][ 'searchengine' ] )
|
||||
if( etat == "" ) {
|
||||
return callback( null, JSON.parse( JSON.stringify( param ) ) );
|
||||
} else {
|
||||
return callback( etat, null );
|
||||
}
|
||||
}
|
||||
utils.levenshtein = ( a, b ) => {
|
||||
if( a.length === 0 ) return b.length;
|
||||
if( b.length === 0 ) return a.length;
|
||||
let tmp, i, j, prev, val, row;
|
||||
// swap to save some memory O(min(a,b)) instead of O(a)
|
||||
if( a.length > b.length ) {
|
||||
tmp = a;
|
||||
a = b;
|
||||
b = tmp;
|
||||
}
|
||||
row = Array( a.length + 1 );
|
||||
// init the row
|
||||
for( i = 0; i <= a.length; i++ ) {
|
||||
row[ i ] = i;
|
||||
}
|
||||
// fill in the rest
|
||||
for( i = 1; i <= b.length; i++ ) {
|
||||
prev = i;
|
||||
for( j = 1; j <= a.length; j++ ) {
|
||||
if( b[ i - 1 ] === a[ j - 1 ] ) {
|
||||
val = row[ j - 1 ]; // match
|
||||
} else {
|
||||
val = Math.min( row[ j - 1 ] + 1, // substitution
|
||||
Math.min( prev + 1, // insertion
|
||||
row[ j ] + 1 ) ); // deletion
|
||||
}
|
||||
row[ j - 1 ] = prev;
|
||||
prev = val;
|
||||
}
|
||||
row[ a.length ] = prev;
|
||||
}
|
||||
return row[ a.length ];
|
||||
};
|
||||
utils.testinarray = ( array, arrayreferent ) => {
|
||||
// au moins un element de array existe dans arryreferent
|
||||
let exist = false;
|
||||
if( arrayreferent ) {
|
||||
//console.log('arrrrrrrrrrrrrrr', arrayreferent)
|
||||
array.forEach( e => {
|
||||
//console.log(e)
|
||||
if( arrayreferent.includes( e ) ) exist = true
|
||||
} )
|
||||
}
|
||||
return exist
|
||||
};
|
||||
/*
|
||||
DIRECTORY
|
||||
*/
|
||||
const isDirectory = source => fs.lstatSync( source )
|
||||
.isDirectory();
|
||||
const getDirectories = source => fs.readdirSync( source )
|
||||
.map( name => path.join( source, name ) )
|
||||
.filter( isDirectory );
|
||||
module.exports = utils;
|
||||
Reference in New Issue
Block a user