<commit_before>#!/usr/bin/env node --harmony

var program = require('commander');
var fs = require('fs');
var chalk = require('chalk');
var JSONStream = require('JSONStream');
var crypto = require('crypto');

program
    .version('0.0.1')
    .option('-i, --input <input>', 'path/to/input.json')
    .option('-o, --output <output>', 'path/to/output.json')
    .option('--stripModels <stripModels>', 
    	'Strip model name for these models. e.g. Anonymous Pump', list)
    .option('--stripSNs <stripSNs>', 
    	'Strip serial number for these models.', list)
    .option('--leaveModels <leaveModels>', 
    	'Leave model for these models. Takes precedence over strip.', list)
    .option('--leaveSNs <leaveSNs>', 
    	'Leave serial number for these models. Takes precedence over strip.', list)
    .option('--stripAll <stripAll>', 
    	'Strip all of the data, except for what is explicitly left.', false)
    .option('--removeTypes <removeTypes>',
    	'Remove these data types.', list)
    .option('--leaveTypes <leaveTypes>',
    	'Leave these data types. Takes precedence over removal.', list)
    .option('--removeAll <removeAll>',
    	'Remove all data types, except for what is explicitly left.', false)
    .option('--hashIDs <hashIDs>',
    	'Pass IDs (such as _groupid and uploadId) through a one-way hash.', false)
    .option('--removeSource <removeSource>',
    	'Remove the source of the data, e.g. carelink.', false)
    .parse(process.argv);

checkOptions();
printOptions();

var ifs = makeInFileStream();

var jsonStream = JSONStream.parse();

var ofs = makeOutstream();

// Perform the parsing
ifs
	.pipe(jsonStream)
	.on('data', function (data) {
	    var allClean = [];
	    for (var i in data) {
		    if ((program.removeAll
		    	|| program.removeTypes
		    		.indexOf(data[i].type) >= 0)
		    	&& program.leaveTypes
		    		.indexOf(data[i].type) < 0) {
		    	// Do NOT add this event to output
		    	continue;
		    }

		    var cleanData = data[i];

		    var deviceId = splitDeviceId(data[i].deviceId);
		    var deviceComp = deviceId[0];

		    if ((program.stripAll
		    	|| program.stripModels
		    		.indexOf(deviceComp) >= 0)
		    	&& program.leaveModels
		    		.indexOf(deviceComp) < 0) {
		    	deviceId[0]=cleanData.type + ' device';
		    }

		    if ((program.stripAll
		    	|| program.stripSNs
		    		.indexOf(deviceComp) >= 0)
		    	&& program.leaveSNs
		    		.indexOf(deviceComp) < 0) {
		    	deviceId[1]='Serial Number';
		    }

		    cleanData.deviceId = deviceId.join('-');

		    if (program.hashIDs) {
		    	cleanData.hash_groupId = 
		    		crypto.createHash('sha256')
		    			.update(cleanData._groupId.toString())
		    			.digest('hex');
		    	delete cleanData._groupId;
		    	cleanData.hash_uploadId = 
		    		crypto.createHash('sha256')
		    			.update(cleanData.uploadId.toString())
		    			.digest('hex');
		    	delete cleanData.uploadId;
		    }

		    if (program.removeSource) {
		    	delete cleanData.source;
		    }

		    allClean.push(JSON.stringify(cleanData));
	    }
	    var cleanStr = '[' + allClean.join(',') + ']\n';

	    writeToOutstream(ofs, cleanStr);
	})
	.on('end', function() {
		console.log(chalk.yellow.bold('Done writing to output.'));
	});


function list(val) {
	return val
		.split(',');
}

function splitDeviceId(deviceId) {
	var retlist=deviceId.split('-');
	if (retlist.length === 1) {
		// Probably split by '_'
		retlist=deviceId.split('_');
	} else if (retlist.length > 2) {
		// Split into more than two pieces
		var model=retlist.slice(0,-1).join('-');
		var serial=retlist[retlist.length-1];
		retlist=[];
		retlist.push(model);
		retlist.push(serial);
	}
	// Index 0 has model, 1 has serial
	return retlist;
}

function checkOptions() {
	if (program.stripAll === 'true') {program.stripAll = true;}
	if (program.stripAll === 'false') {program.stripAll = false;}<commit_msg>Refactoring of stripdata.<commit_after>#!/usr/bin/env node --harmony

var program = require('commander');
var fs = require('fs');
var chalk = require('chalk');
var JSONStream = require('JSONStream');
var crypto = require('crypto');

program
    .version('0.0.1')
    .option('-i, --input <input>', 'path/to/input.json')
    .option('-o, --output <output>', 'path/to/output.json')
    .option('--stripModels <stripModels>', 
    	'Strip model name for these models. e.g. Anonymous Pump', list)
    .option('--stripSNs <stripSNs>', 
    	'Strip serial number for these models.', list)
    .option('--leaveModels <leaveModels>', 
    	'Leave model for these models. Takes precedence over strip.', list)
    .option('--leaveSNs <leaveSNs>', 
    	'Leave serial number for these models. Takes precedence over strip.', list)
    .option('--stripAll <stripAll>', 
    	'Strip all of the data, except for what is explicitly left.', false)
    .option('--removeTypes <removeTypes>',
    	'Remove these data types.', list)
    .option('--leaveTypes <leaveTypes>',
    	'Leave these data types. Takes precedence over removal.', list)
    .option('--removeAll <removeAll>',
    	'Remove all data types, except for what is explicitly left.', false)
    .option('--hashIDs <hashIDs>',
    	'Pass IDs (such as _groupid and uploadId) through a one-way hash.', false)
    .option('--removeSource <removeSource>',
    	'Remove the source of the data, e.g. carelink.', false)
    .parse(process.argv);

checkOptions();
printOptions();

var ifs = makeInFileStream();

var jsonStream = JSONStream.parse();

var ofs = makeOutstream();

// Perform the parsing
ifs
	.pipe(jsonStream)
	.on('data', function (chunk) {
	    var allClean = [];
	    for (var i in chunk) {
		    if ((program.removeAll
		    	|| program.removeTypes
		    		.indexOf(chunk[i].type) >= 0)
		    	&& program.leaveTypes
		    		.indexOf(chunk[i].type) < 0) {
		    	// Do NOT add this event to output
		    	continue;
		    }

		    // Make data into object so it is
		    // passed by reference.
		    var cleanData = {val: chunk[i]};

		    stripModelAndSNForData(cleanData);

		    hashIDsForData(cleanData);

		    removeSourceForData(cleanData);

		    allClean.push(JSON.stringify(cleanData.val));
	    }
	    var cleanStr = '[' + allClean.join(',') + ']\n';

	    writeToOutstream(ofs, cleanStr);
	})
	.on('end', function() {
		console.log(chalk.yellow.bold('Done writing to output.'));
	});


function list(val) {
	return val
		.split(',');
}

function splitDeviceId(deviceId) {
	var retlist=deviceId.split('-');
	if (retlist.length === 1) {
		// Probably split by '_'
		retlist=deviceId.split('_');
	} else if (retlist.length > 2) {
		// Split into more than two pieces
		var model=retlist.slice(0,-1).join('-');
		var serial=retlist[retlist.length-1];
		retlist=[];
		retlist.push(model);
		retlist.push(serial);
	}
	// Index 0 has model, 1 has serial
	return retlist;
}

// Because data is an object, this passes data
// by reference.
function stripModelAndSNForData(data) {
	var deviceId = splitDeviceId(data.val.deviceId);
    var deviceComp = deviceId[0];

    if ((program.stripAll
    	|| program.stripModels
    		.indexOf(deviceComp) >= 0)
    	&& program.leaveModels
    		.indexOf(deviceComp) < 0) {
    	deviceId[0]=data.val.type + ' device';
    }

    if ((program.stripAll
    	|| program.stripSNs
    		.indexOf(deviceComp) >= 0)
    	&& program.leaveSNs
    		.indexOf(deviceComp) < 0) {
    	deviceId[1]='Serial Number';
    }

    data.val.deviceId = deviceId.join('-');
}

// Because data is an object, this passes data
// by reference.
function hashIDsForData(data) {
	if (program.hashIDs) {
    	data.val.hash_groupId = 
    		crypto.createHash('sha256')
    			.update(data.val._groupId.toString())
    			.digest('hex');
    	delete data.val._groupId;
    	data.val.hash_uploadId = 
    		crypto.createHash('sha256')
    			.update(data.val.uploadId.toString())
    			.digest('hex');
    	delete data.val.uploadId;
    }
}

// Because data is an object, this passes data
// by reference.
function removeSourceForData(data) {
	if (program.removeSource) {
    	delete data.val.source;
    }
}

function checkOptions() {
	if (program.stripAll === 'true') {program.stripAll = true;}
	if (program.stripAll === 'false') {program.stripAll = false;}
{'fraction_non_alphanumeric': 0.15649738156132018, 'fraction_numerical': 0.00608939227865059, 'mean_word_length': 2.9461797212878422, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3337719', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2682, 'n_words': 657}