<commit_before>var fs = require('fs');
var _ = require('underscore');
var async = require('async');
var mkdirp = require('mkdirp');
var path = require('path');
var archiver = require('archiver');
var asyncEachObject = require('async-each-object');

function FileSystemHelper() {

};

/**
 * Generates JSON files for current spreadsheet
 * @param  {Object}   localisation Object with all localisation data
 * @param  {Function} callback
 */
FileSystemHelper.prototype.generateJsonFiles = function(localisation, callback) {
	var time = new Date();
	var prefix = '' + time.getDate() + time.getMonth() + time.getFullYear() + 
				'_' + time.getHours() + time.getMinutes() + time.getSeconds();
	var root = __dirname + '/localisation/' + prefix;
	async.eachObject(localisation, function(value, lang, done) {
		var value = _.filter(value, function(it) {
			return it.translation;
		});
		var vis = _.where(value, {project: 'VIS'});
		var csb = _.where(value, {project: 'CSB'});
		var visContent = {
			soccer: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			basketball: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			icehockey: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			volleyball: {
				stats: {},
				eventCodes: {},
				strings: {}
			}
		};
		var csbContent = {};
		_.each(vis, function(item) {
			var parsedKey = item.key.split('->');
			var section = '';
			var sport = '';
			var key = parsedKey[2];
			if(parsedKey[1] === 'STR') {
				section = 'strings';
			} else if(parsedKey[1] === 'CODES') {
				section = 'eventCodes';
			} else {
				section = 'stats';
			}
			if(parsedKey[0] === 'COMMON') {
				sport = 'all';
			} else if(parsedKey[0] === 'FB') {
				sport = 'soccer';
			} else if(parsedKey[0] === 'BB') {
				sport = 'basketball';
			} else if(parsedKey[0] === 'IH') {
				sport = 'icehockey';<commit_msg>Fixed key parsing with '->' symbols<commit_after>var fs = require('fs');
var _ = require('underscore');
var async = require('async');
var mkdirp = require('mkdirp');
var path = require('path');
var archiver = require('archiver');
var asyncEachObject = require('async-each-object');

function FileSystemHelper() {

};

/**
 * Generates JSON files for current spreadsheet
 * @param  {Object}   localisation Object with all localisation data
 * @param  {Function} callback
 */
FileSystemHelper.prototype.generateJsonFiles = function(localisation, callback) {
	var time = new Date();
	var prefix = '' + time.getDate() + time.getMonth() + time.getFullYear() + 
				'_' + time.getHours() + time.getMinutes() + time.getSeconds();
	var root = __dirname + '/localisation/' + prefix;
	async.eachObject(localisation, function(value, lang, done) {
		var value = _.filter(value, function(it) {
			return it.translation;
		});
		var vis = _.where(value, {project: 'VIS'});
		var csb = _.where(value, {project: 'CSB'});
		var visContent = {
			soccer: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			basketball: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			icehockey: {
				stats: {},
				eventCodes: {},
				strings: {}
			},
			volleyball: {
				stats: {},
				eventCodes: {},
				strings: {}
			}
		};
		var csbContent = {};
		_.each(vis, function(item) {
			var parsedKey = item.key.split('->');
			var section = '';
			var sport = '';
			var key = parsedKey.splice(2).join('->');
			if(parsedKey[1] === 'STR') {
				section = 'strings';
			} else if(parsedKey[1] === 'CODES') {
				section = 'eventCodes';
			} else {
				section = 'stats';
			}
			if(parsedKey[0] === 'COMMON') {
				sport = 'all';
			} else if(parsedKey[0] === 'FB') {
				sport = 'soccer';
			} else if(parsedKey[0] === 'BB') {
				sport = 'basketball';
			} else if(parsedKey[0] === 'IH') {
				sport = 'icehockey';
{'fraction_non_alphanumeric': 0.1875993640699523, 'fraction_numerical': 0.0037095919448860626, 'mean_word_length': 3.2995444191343966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6059057', 'n_tokens_mistral': 1502, 'n_tokens_neox': 1292, 'n_words': 287}