<commit_before>//This file organize data, and store a cache of it

var fs = require('fs');
var panneaux = require('../dataSources/panneaux.js');
var voie_pub = require('../dataSources/voie_pub.js');

var dataCache = exports.dataCache = {};

function sources() {
	return {'panneaux' : panneaux, 'voie_pub' : voie_pub};
}

function parseDataForKey(key, callback) {
	var path = fullPathOfFileOfKey(key);

	fs.exists(path, function (exist) {
		if (exist) {
			dataCache[key] = JSON.parse(fs.readFileSync(path));
			console.log("Loaded " + key);
			<commit_msg>Fix crash when json we exported is not valid<commit_after>//This file organize data, and store a cache of it

var fs = require('fs');
var panneaux = require('../dataSources/panneaux.js');
var voie_pub = require('../dataSources/voie_pub.js');

var dataCache = exports.dataCache = {};

function sources() {
	return {'panneaux' : panneaux, 'voie_pub' : voie_pub};
}

function parseDataForKey(key, callback) {
	var path = fullPathOfFileOfKey(key);

	fs.exists(path, function (exist) {
		if (exist) {
			try {
				dataCache[key] = JSON.parse(fs.readFileSync(path));
				console.log("Loaded " + key);
			}
			catch (err) {
				//The json we have on disk is invalid, delete the file
				//That will force the re-download of the data next update
				fs.unlinkSync(path);
				console.log("Could lot load " + key);	
			}

{'fraction_non_alphanumeric': 0.14557425018288223, 'fraction_numerical': 0.0, 'mean_word_length': 4.010989010989011, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5240844', 'n_tokens_mistral': 500, 'n_tokens_neox': 462, 'n_words': 125}