<commit_before>'use strict';
module.exports = (input) => {
    return input
        .split(/[\r\n]+/)
        .filter(isNotEmptyLine)
        .filter(isNotCommentLine)
        .map(parseMatcher);
};

//TODO:
//Can the line be split up into sections beforeprocessing it?
//If it can, then this really should be a reduce.
//If not, does this version of node have tail-call ellimination? ...apparently no
//might be best to swap back to loop if it doesn't
function parseLine(matcher, resultHolder, lineNum) {
    // Look for start of marked up section
    const iBegin = matcher.indexOf("[");

    if (iBegin === -1) {
        // No marked up sections remaining
        resultHolder.result.push({ fuzzy: matcher });
        return resultHolder;
    } else if (iBegin > 0) {
        // Add the fuzzy before the marked up section
        resultHolder.result.push({ fuzzy: matcher.substr(0, iBegin) });
    }

    // Look for end of marked up section
    const type = matcher[iBegin+1];
    matcher = matcher.substr(iBegin);
    const iEnd = matcher.indexOf("]");

    if (iEnd === -1) { throw Error(`Syntax Error: Unterminated '[' on line ${lineNum}`); }

    const markedUpSection = matcher.substr(2, iEnd-2);

    if (type === "+") {
        resultHolder.result.push({ invariant: markedUpSection});
        resultHolder.hasInvariant = true;
    } else if (type === "-") {
        resultHolder.result.push({ disallowed: markedUpSection });
    } else if (type === "v") {
        resultHolder.result.push({ version: trimFromEnd(markedUpSection, /[0-9\-_\.]/) });
    } else {
        throw Error(`Syntax Error: Invalid markup '[${type}...]' on line ${lineNum}`);
    }

    const nextMatcher = matcher.substr(iEnd+1);

    if(nextMatcher === "") {
        return resultHolder;
    } else {
        return parseLine(nextMatcher, resultHolder, lineNum);
    }
}

function parseMatcher(line, lineNum) {

    const topRegex = /\s*([^\s]+)\s+([^:\s]+)\s*:\s*(.*)/;

    const sections = line.match(topRegex);
    if (!sections) { throw Error(`Syntax Error at top level (brand model:matcher) on line ${lineNum}`); }
    const brand = sections[1];
    const model = sections[2];
    const matcher = sections[3];
    const result = [ { brand, model } ];
    const hasInvariant = false;

    const resultHolder = parseLine(matcher, {result, hasInvariant}, lineNum);

    if (!resultHolder.hasInvariant) {throw Error(`Error: Matcher has no invariants on line ${lineNum}`); }
    return resultHolder.result;
};

//TODO:
//Another possible split then reduce?
function trimFromEnd(value, toTrim) {
    if(!value[value.length-1].match(toTrim)) {return value; }
    else {return trimFromEnd(value.substr(0, value.length-1), toTrim);}
};

function isNotEmptyLine(line) {
    const emptyLineRegex = /^\s*$/;
    return line.match(emptyLineRegex) == null;
};

function isNotCommentLine(line) {
    const commentLineRegex = /^\s*#.*$/;
    return line.match(commentLineRegex) == null;
};<commit_msg>Fix issue with version numbers crashing the parser<commit_after>'use strict';
module.exports = (input) => {
    return input
        .split(/[\r\n]+/)
        .filter(isNotEmptyLine)
        .filter(isNotCommentLine)
        .map(parseMatcher);
};

//TODO:
//Can the line be split up into sections beforeprocessing it?
//If it can, then this really should be a reduce.
//If not, does this version of node have tail-call ellimination? ...apparently no
//might be best to swap back to loop if it doesn't
function parseLine(matcher, resultHolder, lineNum) {
    // Look for start of marked up section
    const iBegin = matcher.indexOf("[");

    if (iBegin === -1) {
        // No marked up sections remaining
        resultHolder.result.push({ fuzzy: matcher });
        return resultHolder;
    } else if (iBegin > 0) {
        // Add the fuzzy before the marked up section
        resultHolder.result.push({ fuzzy: matcher.substr(0, iBegin) });
    }

    // Look for end of marked up section
    const type = matcher[iBegin+1];
    matcher = matcher.substr(iBegin);
    const iEnd = matcher.indexOf("]");

    if (iEnd === -1) { throw Error(`Syntax Error: Unterminated '[' on line ${lineNum}`); }

    const markedUpSection = matcher.substr(2, iEnd-2);

    if (type === "+") {
        resultHolder.result.push({ invariant: markedUpSection});
        resultHolder.hasInvariant = true;
    } else if (type === "-") {
        resultHolder.result.push({ disallowed: markedUpSection });
    } else if (type === "v") {
        resultHolder.result.push({ version: trimFromEnd(markedUpSection, /[0-9\-_\.]/) });
    } else {
        throw Error(`Syntax Error: Invalid markup '[${type}...]' on line ${lineNum}`);
    }

    const nextMatcher = matcher.substr(iEnd+1);

    if(nextMatcher === "") {
        return resultHolder;
    } else {
        return parseLine(nextMatcher, resultHolder, lineNum);
    }
}

function parseMatcher(line, lineNum) {

    const topRegex = /\s*([^\s]+)\s+([^:\s]+)\s*:\s*(.*)/;

    const sections = line.match(topRegex);
    if (!sections) { throw Error(`Syntax Error at top level (brand model:matcher) on line ${lineNum}`); }
    const brand = sections[1];
    const model = sections[2];
    const matcher = sections[3];
    const result = [ { brand, model } ];
    const hasInvariant = false;

    const resultHolder = parseLine(matcher, {result, hasInvariant}, lineNum);

    if (!resultHolder.hasInvariant) {throw Error(`Error: Matcher has no invariants on line ${lineNum}`); }
    return resultHolder.result;
};

//TODO:
//Another possible split then reduce?
function trimFromEnd(value, toTrim) {
    if(!value) {return;}
    if(!value[value.length-1].match(toTrim)) {return value;}
    else {return trimFromEnd(value.substr(0, value.length-1), toTrim);}
};

function isNotEmptyLine(line) {
    const emptyLineRegex = /^\s*$/;
    return line.match(emptyLineRegex) == null;
};

function isNotCommentLine(line) {
    const commentLineRegex = /^\s*#.*$/;
    return line.match(commentLineRegex) == null;
};
{'fraction_non_alphanumeric': 0.15172184328730662, 'fraction_numerical': 0.005323573448677425, 'mean_word_length': 3.1093643198906356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1593036', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1823, 'n_words': 525}