<commit_before>/*
** Module: aireas2json
**
**
**
**
*/

// **********************************************************************************

var fs 		= require('fs');

var aireasFolder, aireasUrl, aireasFileName, aireasLocalPathRoot, fileFolder, 
	tmpFolder, tmpFolderName, localPath, fileFolderName, resultsFolder, resultsFolderName;
var dataRecords;

// **********************************************************************************

module.exports = {

	init: function (options) {
		aireasLocalPathRoot = options.systemFolderParent+'/aireas/';
		fileFolderName 		= 'aireas';
		tmpFolderName 		= 'tmp';
		resultsFolderName 	= 'results';

		aireasFileName 		= 'aireas.txt';
		aireasFileNameOutput= 'aireas.json';

		aireasFolder 		= aireasLocalPathRoot 	+ fileFolderName 	+ "/";
		tmpFolder 			= aireasFolder 			+ tmpFolderName 	+ "/";
		resultsFolder 		= aireasFolder 			+ resultsFolderName + "/";

		// create subfolders
		try {fs.mkdirSync(resultsFolder );} catch (e) {};//console.log('ERROR: no tmp folder found, batch run aborted.'); return } ;

	    localPath 		= tmpFolder + aireasFileName;
    	var _datFile	= fs.readFileSync(localPath);
    	console.log("Local AiREAS data: " + localPath );


		var i, j, _dataRecord, _waardeDataRecord, inpRecordArray,
			inpRecordPM1, inpRecordPM25, inpRecordPM10, 
			inpRecordUFP, inpRecordOZON, inpRecordHUM, inpRecordCELC;

		var tmp0 = _datFile.toString();
		var dateRetrievedLength = tmp0.indexOf(' ');
		var dateRetrieved = tmp0.substring(0, dateRetrievedLength);
		var dateRetrievedDate = Date.parse(dateRetrieved);
		console.log('x'+dateRetrieved+'x');
		var tmp1 = tmp0.substr(dateRetrievedLength+1);
		var tmp2 = tmp1.replace(/<p>/g,'');
		var tmp3 = tmp2.replace(/<\/p>/g,'');
		//var tmpArray = tmp3.split(')');
		var tmpArray = tmp3.split(']');
		

		dataRecords	= [];

		for(i=1;i<tmpArray.length-1;i++) {  // start i=1 !!

//			inpRecordArray 		= tmpArray[i].split(':(');
			inpRecordArray 		= tmpArray[i].split('[');

			_dataRecord			= {};
//			_dataRecord.airBox	= inpRecordArray[0];

			inpMetingenArray 	= inpRecordArray[1].split(',');
	
			_waardeDataRecord 	= [];
			for(j=0;j<inpMetingenArray.length;j++) {
				_waardeDataRecord[j] = inpMetingenArray[j];// parseFloat(inpMetingenArray[j]);
			}
						
			_dataRecord.airbox 	= _waardeDataRecord[0];
			_dataRecord.retrievedDate 	= dateRetrieved;
			_dataRecord.measureDate 	= _waardeDataRecord[1];
			_dataRecord.gpsLat 	= _waardeDataRecord[10];
			_dataRecord.gpsLng 	= _waardeDataRecord[11];
			_dataRecord.lat 	= this.convertGPS2LatLng(_waardeDataRecord[10]);
			_dataRecord.lng 	= this.convertGPS2LatLng(_waardeDataRecord[11]);
			_dataRecord.PM1 	= _waardeDataRecord[2];
			_dataRecord.PM25 	= _waardeDataRecord[3];
			_dataRecord.PM10 	= _waardeDataRecord[4];<commit_msg>new record layout csv<commit_after>/*
** Module: aireas2json
**
**
**
**
*/

// **********************************************************************************

var fs 		= require('fs');

var aireasFolder, aireasUrl, aireasFileName, aireasLocalPathRoot, fileFolder, 
	tmpFolder, tmpFolderName, localPath, fileFolderName, resultsFolder, resultsFolderName;
var dataRecords;

// **********************************************************************************

module.exports = {

	init: function (options) {
		aireasLocalPathRoot = options.systemFolderParent+'/aireas/';
		fileFolderName 		= 'aireas';
		tmpFolderName 		= 'tmp';
		resultsFolderName 	= 'results';

		aireasFileName 		= 'aireas.txt';
		aireasFileNameOutput= 'aireas.json';

		aireasFolder 		= aireasLocalPathRoot 	+ fileFolderName 	+ "/";
		tmpFolder 			= aireasFolder 			+ tmpFolderName 	+ "/";
		resultsFolder 		= aireasFolder 			+ resultsFolderName + "/";

		// create subfolders
		try {fs.mkdirSync(resultsFolder );} catch (e) {};//console.log('ERROR: no tmp folder found, batch run aborted.'); return } ;

	    localPath 		= tmpFolder + aireasFileName;
    	var _datFile	= fs.readFileSync(localPath);
    	console.log("Local AiREAS data: " + localPath );


		var i, j, _dataRecord, _waardeDataRecord, inpRecordArray,
			inpRecordPM1, inpRecordPM25, inpRecordPM10, 
			inpRecordUFP, inpRecordOZON, inpRecordHUM, inpRecordCELC;

		var tmp0 = _datFile.toString();
		var dateRetrievedLength = tmp0.indexOf(' ');
		var dateRetrieved = tmp0.substring(0, dateRetrievedLength);
		var dateRetrievedDate = Date.parse(dateRetrieved);
		console.log('x'+dateRetrieved+'x');
		var tmp1 = tmp0.substr(dateRetrievedLength+1);
		//var tmp2 = tmp1.replace(/<p>/g,'');
		//var tmp3 = tmp2.replace(/<\/p>/g,'');
		//var tmpArray = tmp3.split(')');
		var tmp2 = tmp1.replace(/\[/,'');
		var tmpArray = tmp3.split('],');
		

		dataRecords	= [];

		for(i=1;i<tmpArray.length-1;i++) {  // start i=1 !!

//			inpRecordArray 		= tmpArray[i].split(':(');
			inpRecordArray 		= tmpArray[i].split('[');

			_dataRecord			= {};
//			_dataRecord.airBox	= inpRecordArray[0];

			inpMetingenArray 	= inpRecordArray[1].split(',');
	
			_waardeDataRecord 	= [];
			for(j=0;j<inpMetingenArray.length;j++) {
				_waardeDataRecord[j] = inpMetingenArray[j];// parseFloat(inpMetingenArray[j]);
			}
						
			_dataRecord.airbox 	= _waardeDataRecord[0];
			_dataRecord.retrievedDate 	= dateRetrieved;
			_dataRecord.measureDate 	= _waardeDataRecord[1];
			_dataRecord.gpsLat 	= _waardeDataRecord[10];
			_dataRecord.gpsLng 	= _waardeDataRecord[11];
			_dataRecord.lat 	= this.convertGPS2LatLng(_waardeDataRecord[10]);
			_dataRecord.lng 	= this.convertGPS2LatLng(_waardeDataRecord[11]);
			_dataRecord.PM1 	= _waardeDataRecord[2];
			_dataRecord.PM25 	= _waardeDataRecord[3];
			_dataRecord.PM10 	= _waardeDataRecord[4];
{'fraction_non_alphanumeric': 0.19575596816976126, 'fraction_numerical': 0.016268788682581786, 'mean_word_length': 4.534246575342466, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1492683', 'n_tokens_mistral': 2268, 'n_tokens_neox': 2028, 'n_words': 305}