<commit_before>function CSVtoArray(text) {
    var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
    var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
    // Return NULL if input string is not well formed CSV string.
    if (!re_valid.test(text)) {
        console.warn("CSVtoArray: Invalid csv text.\n\nSee http://stackoverflow.com/questions/8493195/how-can-i-parse-a-csv-string-with-javascript for help.");
        console.warn(text);
        return null;
    }
    var a = [];                     // Initialize array to receive values.
    text.replace(re_value, // "Walk" the string using replace with callback.
        function(m0, m1, m2, m3) {
            // Remove backslash from \' in single quoted values.
            if      (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
            // Remove backslash from \" in double quoted values.
            else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
            else if (m3 !== undefined) a.push(m3);
            return ''; // Return empty string.
        });
    // Handle special case of empty last value.
    if (/,\s*$/.test(text)) a.push('');
    return a;
};

function getMedian(values) {

    values.sort( function(a,b) {return a - b;} );

    var half = Math.floor(values.length/2);

    if(values.length % 2)
        return values[half];
    else
	return values[half-1];
//        return (values[half-1] + values[half]) / 2.0;
}

function handleValidationForURL(url) {
	$.getJSON("http://odinprac.theodi.org/CSV_Dataset_Validator/get_data.php?url="+url, function(json) {
		// Loading percentage
		console.log(json);
		percent = Math.floor((json.completed / json.file_count) * 100);
		$('progress').attr('value',percent);
		if (json.completed < json.file_count) {
			setTimeout(function(){handleValidationForURL(url);},500);
		}
		processFileProblems(json);
		processColumnTitles(json);
		processCSVLintResult(json);
		processLineNumbers(json);
	})
	.error(function() {
        	//console.log("error fetching data");
		setTimeout(function(){handleValidationForURL(url);},500);
   	});
}
function getDataFromURL(url) {
	$.getJSON(url, function(json) {
		$( "#dataset-title" ).html(json.title);
		$( "#dataset-description" ).html(json.notes);
		$( "#result" ).fadeIn("slow",function() {
			handleValidationForURL(url);
		});
	})
	.error(function() {
        	console.log("error fetching data");
		$( "#front-page" ).fadeIn("slow",function() {
		});
		//setTimeout(function(){handleValidationForURL(url);},2000);
   	});
	
}

function processCSVLintResult(json) {
	invalid_count = 0;	
	$("#invalid-csvlint-detail-table").html('<tr><th>File</th><th class="csvlint-error">Errors</th><th class="csvlint-warning">Warnings</th><th class="csvlint-info">Info</th><th>Status</th></tr>');
	for (var file_num in json.csvlintData) {
		current = json.csvlintData[file_num];
		if (current.state == "invalid") {
			invalid_count += 1;
			$("#csvlint-file-count").html(invalid_count);
		}
		if (invalid_count > 0) {
                	$('#csvlint-file-count').css('background-color', '#f2dede');
		}
		file_description = json.files[file_num]["description"];
		result = "<tr><td class='csvlint-ok'><a href='"+current.url+"' target='_blank'>" + file_description + "</a></td>";
		if (current.error_count > 0) { aclass="csvlint-error"; } else { aclass="csvlint-ok"; }
		result += "<td class='"+aclass+"'>" + current.error_count + "</td>";  
		if (current.warning_count > 0) { aclass="csvlint-warning"; } else { aclass="csvlint-ok"; }<commit_msg>Updated control script to work in error, not ideal but it restores service.<commit_after>function CSVtoArray(text) {
    var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
    var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
    // Return NULL if input string is not well formed CSV string.
    if (!re_valid.test(text)) {
        console.warn("CSVtoArray: Invalid csv text.\n\nSee http://stackoverflow.com/questions/8493195/how-can-i-parse-a-csv-string-with-javascript for help.");
        console.warn(text);
        return null;
    }
    var a = [];                     // Initialize array to receive values.
    text.replace(re_value, // "Walk" the string using replace with callback.
        function(m0, m1, m2, m3) {
            // Remove backslash from \' in single quoted values.
            if      (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
            // Remove backslash from \" in double quoted values.
            else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
            else if (m3 !== undefined) a.push(m3);
            return ''; // Return empty string.
        });
    // Handle special case of empty last value.
    if (/,\s*$/.test(text)) a.push('');
    return a;
};

function getMedian(values) {

    values.sort( function(a,b) {return a - b;} );

    var half = Math.floor(values.length/2);

    if(values.length % 2)
        return values[half];
    else
	return values[half-1];
//        return (values[half-1] + values[half]) / 2.0;
}

function handleValidationForURL(url) {
	$.getJSON("http://odinprac.theodi.org/CSV_Dataset_Validator/get_data.php?url="+url, function(json) {
		// Loading percentage
		console.log(json);
		$( "#dataset-title" ).html(json.title);
		$( "#dataset-description" ).html(json.notes);
		percent = Math.floor((json.completed / json.file_count) * 100);
		$('progress').attr('value',percent);
		if (json.completed < json.file_count) {
			setTimeout(function(){handleValidationForURL(url);},500);
		}
		processFileProblems(json);
		processColumnTitles(json);
		processCSVLintResult(json);
		processLineNumbers(json);
	})
	.error(function() {
        	//console.log("error fetching data");
		setTimeout(function(){handleValidationForURL(url);},500);
   	});
}
function getDataFromURL(url) {
	$.getJSON(url, function(json) {
		$( "#dataset-title" ).html(json.title);
		$( "#dataset-description" ).html(json.notes);
		$( "#result" ).fadeIn("slow",function() {
			handleValidationForURL(url);
		});
	})
	.error(function() {
        	console.log("error fetching data");
		$( "#result" ).fadeIn("slow",function() {
			handleValidationForURL(url);
		});
		//$( "#front-page" ).fadeIn("slow",function() {
		//});
		//setTimeout(function(){handleValidationForURL(url);},2000);
   	});
	
}

function processCSVLintResult(json) {
	invalid_count = 0;	
	$("#invalid-csvlint-detail-table").html('<tr><th>File</th><th class="csvlint-error">Errors</th><th class="csvlint-warning">Warnings</th><th class="csvlint-info">Info</th><th>Status</th></tr>');
	for (var file_num in json.csvlintData) {
		current = json.csvlintData[file_num];
		if (current.state == "invalid") {
			invalid_count += 1;
			$("#csvlint-file-count").html(invalid_count);
		}
		if (invalid_count > 0) {
                	$('#csvlint-file-count').css('background-color', '#f2dede');
		}
		file_description = json.files[file_num]["description"];
		result = "<tr><td class='csvlint-ok'><a href='"+current.url+"' target='_blank'>" + file_description + "</a></td>";
		if (current.error_count > 0) { aclass="csvlint-error"; } else { aclass="csvlint-ok"; }
		result += "<td class='"+aclass+"'>" + current.error_count + "</td>";  
		if (current.warning_count > 0) { aclass="csvlint-warning"; } else { aclass="csvlint-ok"; }
{'fraction_non_alphanumeric': 0.2510922641994346, 'fraction_numerical': 0.01079414032382421, 'mean_word_length': 3.3773903262092237, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5952617', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2745, 'n_words': 439}