<commit_before>
function DiffParser(text)
{
    this.lines = text.split("\n");
    this.currentLine = 0;
}

DiffParser.HEADER_BEGIN = "Index: ";
DiffParser.HEADER_END = "+++ ";
DiffParser.BINARY_HEADER_END = "Binary files ";
DiffParser.PNG_SUFFIX = ".png";
DiffParser.HEADER_PATTERN = /^@@\ \-(\d+),[^+]+\+(\d+)\,\d+\ @@\ ?(.*)/;

DiffParser.prototype.peekLine = function()
{
    return this.lines[this.currentLine];
};

DiffParser.prototype.takeLine = function(type)
{
    var line = this.lines[this.currentLine++];
    if (!type)
        return line;
    if (type == "add" || type == "remove" || type == "both")
        return line.slice(1);
    return line;
};

DiffParser.prototype.haveLines = function()
{
    return this.currentLine != this.lines.length;
};

DiffParser.prototype.nextLineType = function()
{
    var line = this.peekLine();
    if (!line.length)
        return "empty";
    var c = line[0];
    if (c == "@")
        return "header";
    if (c == "+")
        return "add";
    if (c == "-")
        return "remove";
    if (c == "I")
        return "index";
    if (c == " ")
        return "both";
    if (c == "\\")
        return "empty";
    throw new Error("Parse error: Unable to classify line: '{1}'".assign(line));
};

DiffParser.prototype.parseFile = function()
{
    var groups = [];
    var currentGroupType = null;
    var currentGroup = [];
    var currentBeforeLineNumber = 0;
    var currentAfterLineNumber = 0;
    while (this.haveLines()) {
        var type = this.nextLineType();
        if (type == "index" || type == "empty")
            break; // We're done with this file.

        var groupType = type;
        var line = {type: type};

        if (groupType == "header") {
            var matchedHeader = this.takeLine().match(DiffParser.HEADER_PATTERN);
            currentBeforeLineNumber = matchedHeader[1];
            currentAfterLineNumber = matchedHeader[2];
            line.beforeNumber = "@@";
            line.afterNumber = "@@";
            line.text = matchedHeader[3];
        } else {
            line.beforeNumber = currentBeforeLineNumber;
            line.afterNumber = currentAfterLineNumber;
            line.text = this.takeLine(type);

            if (groupType == "remove" || groupType == "both")
                currentBeforeLineNumber++;
            if (groupType == "add" || groupType == "both")
                currentAfterLineNumber++;
        }

        if (groupType == "add" || groupType == "remove")
            groupType = "delta";
        if (groupType != currentGroupType) {<commit_msg>Store the context line range in the diff parser<commit_after>
function DiffParser(text)
{
    this.lines = text.split("\n");
    this.currentLine = 0;
}

DiffParser.HEADER_BEGIN = "Index: ";
DiffParser.HEADER_END = "+++ ";
DiffParser.BINARY_HEADER_END = "Binary files ";
DiffParser.PNG_SUFFIX = ".png";
DiffParser.HEADER_PATTERN = /^@@\ \-(\d+),[^+]+\+(\d+)\,\d+\ @@\ ?(.*)/;

DiffParser.prototype.peekLine = function()
{
    return this.lines[this.currentLine];
};

DiffParser.prototype.takeLine = function(type)
{
    var line = this.lines[this.currentLine++];
    if (!type)
        return line;
    if (type == "add" || type == "remove" || type == "both")
        return line.slice(1);
    return line;
};

DiffParser.prototype.haveLines = function()
{
    return this.currentLine != this.lines.length;
};

DiffParser.prototype.nextLineType = function()
{
    var line = this.peekLine();
    if (!line.length)
        return "empty";
    var c = line[0];
    if (c == "@")
        return "header";
    if (c == "+")
        return "add";
    if (c == "-")
        return "remove";
    if (c == "I")
        return "index";
    if (c == " ")
        return "both";
    if (c == "\\")
        return "empty";
    throw new Error("Parse error: Unable to classify line: '{1}'".assign(line));
};

DiffParser.prototype.parseFile = function()
{
    var groups = [];
    var currentGroupType = null;
    var currentGroup = [];
    var currentBeforeLineNumber = 0;
    var currentAfterLineNumber = 0;
    while (this.haveLines()) {
        var type = this.nextLineType();
        if (type == "index" || type == "empty")
            break; // We're done with this file.

        var groupType = type;
        var line = {
            type: type,
            beforeNumber: 0,
            afterNumber: 0,
            contextLinesStart: 0,
            contextLinesEnd: 0,
            text: "",
        };

        if (groupType == "header") {
            var matchedHeader = this.takeLine().match(DiffParser.HEADER_PATTERN);
            var beforeLineNumber = parseInt(matchedHeader[1], 10);
            var afterLineNumber = parseInt(matchedHeader[2], 10);
            line.contextLinesStart = currentAfterLineNumber;
            line.contextLinesEnd = afterLineNumber - 1;
            line.beforeNumber = "@@";
            line.afterNumber = "@@";
            line.text = matchedHeader[3];
            currentBeforeLineNumber = beforeLineNumber;
            currentAfterLineNumber = afterLineNumber;
        } else {
            line.beforeNumber = currentBeforeLineNumber;
            line.afterNumber = currentAfterLineNumber;
            line.text = this.takeLine(type);

            if (groupType == "remove" || groupType == "both")
                currentBeforeLineNumber++;
            if (groupType == "add" || groupType == "both")
                currentAfterLineNumber++;
        }

        if (groupType == "add" || groupType == "remove")
            groupType = "delta";
        if (groupType != currentGroupType) {
{'fraction_non_alphanumeric': 0.15392876320028637, 'fraction_numerical': 0.004832647216753177, 'mean_word_length': 2.283196239717979, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2050169', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1637, 'n_words': 376}