<commit_before>'use strict';

var isOwnProperty = Object.prototype.hasOwnProperty;

function stripComments(value) {
    var index = value.indexOf('%');

    if (index !== -1) {
        value = value.substr(0, index);
    }

    return value;
}

function trimWhiteSpace(value) {
    return value.trim();
}

function isNonEmpty(value) {
    return Boolean(value);
}

function toPropertyValuePairs(value) {
    var values, result;

    values = value.split(':');
    result = [trimWhiteSpace(values.shift())];

    if (values.length) {
        result.push(trimWhiteSpace(values.join(':')));
    }

    return result;
}

function sortOnFirstIndex(a, b) {
    return b[0] - a[0];
}

function propertyValuePairsToObject(pairs) {
    var value = {};

    pairs.forEach(function (pair) {
        value[pair[0]] = pair[1];
    });

    return value;
}

function textToJSON(value) {
    var lines = value.split('\n'),
        propertyOrValues = {},
        isPropertyValuePair, pairs, values;

    lines = lines
        .map(stripComments)
        .map(trimWhiteSpace)
        .filter(isNonEmpty);

    pairs = lines.map(toPropertyValuePairs);

    pairs.forEach(function (line, index) {
        var currentLineIsPropertyValuePair = line.length === 2;

        if (index === 0) {
            isPropertyValuePair = currentLineIsPropertyValuePair;
        } else {
            if (currentLineIsPropertyValuePair !== isPropertyValuePair) {
                throw new Error(
                    'Both property-value pairs and array values found. ' +
                    'Make sure either exists.'
                );
            }
        }

        if (
            line[0] in propertyOrValues &&
            isOwnProperty.call(propertyOrValues, line[0])
        ) {
            throw new Error(
                'Duplicate data found. ' +
                'Make sure, in objects, no duplicate properties exist, in ' +
                'arrays, no duplicate values.'
            );
        }<commit_msg>Add functionality for custom comment and delimiter tokens<commit_after>'use strict';

var isOwnProperty = Object.prototype.hasOwnProperty;

function stripComments(token) {
    return function (value) {
        var index = value.indexOf(token);

        if (index !== -1) {
            value = value.substr(0, index);
        }

        return value;
    };
}

function trimWhiteSpace(value) {
    return value.trim();
}

function isNonEmpty(value) {
    return Boolean(value);
}

function toPropertyValuePairs(token) {
    return function (value) {
        var values, result;

        values = value.split(token);
        result = [trimWhiteSpace(values.shift())];

        if (values.length) {
            result.push(trimWhiteSpace(values.join(token)));
        }

        return result;
    };
}

function sortOnFirstIndex(a, b) {
    return b[0] - a[0];
}

function propertyValuePairsToObject(pairs) {
    var value = {};

    pairs.forEach(function (pair) {
        value[pair[0]] = pair[1];
    });

    return value;
}

function textToJSON(value, options) {
    var propertyOrValues = {},
        lines, isPropertyValuePair, pairs, values;

    if (!options) {
        options = {};
    }

    lines = value.split('\n')
        .map(stripComments(options.comment || '%'))
        .map(trimWhiteSpace)
        .filter(isNonEmpty);

    pairs = lines.map(toPropertyValuePairs(options.delimiter || ':'));

    pairs.forEach(function (line, index) {
        var currentLineIsPropertyValuePair = line.length === 2;

        if (index === 0) {
            isPropertyValuePair = currentLineIsPropertyValuePair;
        } else {
            if (currentLineIsPropertyValuePair !== isPropertyValuePair) {
                throw new Error(
                    'Both property-value pairs and array values found. ' +
                    'Make sure either exists.'
                );
            }
        }

        if (
            line[0] in propertyOrValues &&
            isOwnProperty.call(propertyOrValues, line[0])
        ) {
            throw new Error(
                'Duplicate data found. ' +
                'Make sure, in objects, no duplicate properties exist, in ' +
                'arrays, no duplicate values.'
            );
        }
{'fraction_non_alphanumeric': 0.11714690599905526, 'fraction_numerical': 0.004723665564478035, 'mean_word_length': 2.062183658712943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6084578', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1173, 'n_words': 270}