<commit_before>// Parse a string of TSV or CSV. Returns a flat array of objects of the form { column: value }
// as well as the column names using `d3.dsv`

var d3 = require("d3");
var each = require("lodash/each");
require("sugar-date");
var parseUtils = require("./parse-utils");
var help = require("./helper.js");
var assign = require("lodash/assign");
var defaults = require("lodash/defaults");
var unique = require("lodash/uniq");
var separators;

// We need this to get the current locale's thousands separator
// Check for localStorage in case we are testing from node
if (typeof(localStorage) !== 'undefined') {
	separators = require("../stores/SessionStore").get("separators");
} else {
	separators = {
		decimal: ".",
		thousands: ","
	};
}

var stripChars = [
	"$",
	"£",
	"€",
	"%"
];

var newLineRegex = /\r\n|\r|\n/;
var parseErrors = [];

function parseDelimInput(input, opts) {
	opts = opts || {};
	var _defaultOpts = defaults(opts, {
		delimiter: parseUtils.detectDelimiter(input),
		type: opts.type,
		inputTZ: "Z"
	});

	if (opts.checkForDate === false) {
		_defaultOpts.type = "ordinal";
	}

	parseErrors = [];
	// create regex of special characters we want to strip out as well as our
	// computed locale-specific thousands separator.
	var _stripCharsStr = stripChars.concat([separators.thousands]).reduce(function(a, b) {
		return a.concat(parseUtils.escapeRegExp(b));
	}, []).join("|");
	var stripCharsRegex = new RegExp(_stripCharsStr, "g");

	var columnNames = input.split(newLineRegex)[0].split(_defaultOpts.delimiter);
	var dsv = d3.dsv(_defaultOpts.delimiter, "text/plain");
	var all_index_types = [];

	var casted_data = cast_data(input, columnNames, stripCharsRegex, _defaultOpts);
	var data = casted_data.data;
	all_index_types = casted_data.indexes;
	var all_entry_values = casted_data.entries;<commit_msg>make sure hasDate is a variable even if null<commit_after>// Parse a string of TSV or CSV. Returns a flat array of objects of the form { column: value }
// as well as the column names using `d3.dsv`

var d3 = require("d3");
var each = require("lodash/each");
require("sugar-date");
var parseUtils = require("./parse-utils");
var help = require("./helper.js");
var assign = require("lodash/assign");
var defaults = require("lodash/defaults");
var unique = require("lodash/uniq");
var separators;

// We need this to get the current locale's thousands separator
// Check for localStorage in case we are testing from node
if (typeof(localStorage) !== 'undefined') {
	separators = require("../stores/SessionStore").get("separators");
} else {
	separators = {
		decimal: ".",
		thousands: ","
	};
}

var stripChars = [
	"$",
	"£",
	"€",
	"%"
];

var newLineRegex = /\r\n|\r|\n/;
var parseErrors = [];

function parseDelimInput(input, opts) {
	var hasDate = null;
	var isNumeric = null;

	opts = opts || {};
	var _defaultOpts = defaults(opts, {
		delimiter: parseUtils.detectDelimiter(input),
		type: opts.type,
		inputTZ: "Z"
	});

	if (opts.checkForDate === false) {
		_defaultOpts.type = "ordinal";
	}

	parseErrors = [];
	// create regex of special characters we want to strip out as well as our
	// computed locale-specific thousands separator.
	var _stripCharsStr = stripChars.concat([separators.thousands]).reduce(function(a, b) {
		return a.concat(parseUtils.escapeRegExp(b));
	}, []).join("|");
	var stripCharsRegex = new RegExp(_stripCharsStr, "g");

	var columnNames = input.split(newLineRegex)[0].split(_defaultOpts.delimiter);
	var dsv = d3.dsv(_defaultOpts.delimiter, "text/plain");
	var all_index_types = [];

	var casted_data = cast_data(input, columnNames, stripCharsRegex, _defaultOpts);
	var data = casted_data.data;
	all_index_types = casted_data.indexes;
	var all_entry_values = casted_data.entries;
{'fraction_non_alphanumeric': 0.15191897654584222, 'fraction_numerical': 0.0026652452025586353, 'mean_word_length': 4.601492537313433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5509046', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1239, 'n_words': 337}