<commit_before>function contains(array, line) {
    for (var i = 0; i < array.length; i++) {
        if (array[i].line === line) {
            return true;
        }
    }
    return false;
}

function updateLineNumbers(array, startPos) {
    for (var i = 0; i < array.length; i++) {
        if (array[i].line >= startPos) {
            array[i].line++;
        }
    }
}

function lineUpText(text1, text2, results) {
    var text1Lines = text1.split('\n');
    var text2Lines = text2.split('\n');

    padBlankLines(text1Lines);
    padBlankLines(text2Lines);

    for (var i = 0; i < text1Lines.length && i < text2Lines.length; i++) {
        if (contains(results.deleted, i) && !contains(results.added, i)) {
            text2Lines.splice(i, 0, '');
            updateLineNumbers(results.added, i);
        } else if (!contains(results.deleted, i) && contains(results.added, i)) {
            text1Lines.splice(i, 0, '');
            updateLineNumbers(results.deleted, i);
        }
    }

    return [text1Lines, text2Lines];
}

function doDiff() {
    var text1 = document.getElementById('original').value;
    var text2 = document.getElementById('edited').value;

    var results = diff(text1, text2);

    var lines = lineUpText(text1, text2, results);

    var text1Lines = lines[0];
    var text2Lines = lines[1];

    var deletedText = '';
    for(var i = 0; i < text1Lines.length; i++) {
        if (contains(results.deleted, i)) {
            deletedText += '<span class="deleted">';
        }
        deletedText += text1Lines[i];
        if (contains(results.deleted, i)) {
            deletedText += '</span>';
        }
        deletedText += '<br>';
    }

    var addedText = '';
    for(i = 0; i < text2Lines.length; i++) {
        if (contains(results.added, i)) {
            addedText += '<span class="inserted">';
        }
        addedText += text2Lines[i];
        if (contains(results.added, i)) {
            addedText += '</span>';
        }
        addedText += "<br>";
    }

    document.getElementById('original_result').innerHTML = deletedText;
    document.getElementById('edited_result').innerHTML = addedText;
}<commit_msg>refactoring, convert tabs to three spaces<commit_after>function contains(array, line) {
    for (var i = 0; i < array.length; i++) {
        if (array[i].line === line) {
            return true;
        }
    }
    return false;
}

function updateLineNumbers(array, startPos) {
    for (var i = 0; i < array.length; i++) {
        if (array[i].line >= startPos) {
            array[i].line++;
        }
    }
}

function lineUpText(text1, text2, results) {
    var text1Lines = text1.split('\n');
    var text2Lines = text2.split('\n');

    padBlankLines(text1Lines);
    padBlankLines(text2Lines);

    for (var i = 0; i < text1Lines.length && i < text2Lines.length; i++) {
        if (contains(results.deleted, i) && !contains(results.added, i)) {
            text2Lines.splice(i, 0, '');
            updateLineNumbers(results.added, i);
        } else if (!contains(results.deleted, i) && contains(results.added, i)) {
            text1Lines.splice(i, 0, '');
            updateLineNumbers(results.deleted, i);
        }
    }

    return [text1Lines, text2Lines];
}

function getStartingPos(results) {
    var firstDelete = results.deleted.length > 0
        ? results.deleted[0].line
        : 0;

    var firstAdd = results.added.length > 0
        ? results.added[0].line
        : 0;

    var firstEdit = Math.min(firstDelete, firstAdd);

    var startingPos = Math.max(0, firstEdit - 10);
    return startingPos;
}

function getEndingPos(results, lines) {
    var lastDelete = results.deleted.length > 0
        ? results.deleted[results.deleted.length - 1].line
        : 0;

    var lastAdd = results.added.length > 0
        ? results.added[results.added.length - 1].line
        : 0;

    var lastEdit = Math.max(lastDelete, lastAdd);

    return Math.min(lines.length, lastEdit + 10);
}

function doDiff() {
    var text1 = document.getElementById('original').value;
    var text2 = document.getElementById('edited').value;

    var results = diff(text1, text2);

    var lines = lineUpText(text1, text2, results);

    var text1Lines = lines[0];
    var text2Lines = lines[1];

    var startingPos = getStartingPos(results);
    var text1EndingPos = getEndingPos(results, text1Lines);
    var text2EndingPos = getEndingPos(results, text2Lines);

    var deletedText = '';
    for(var i = startingPos; i < text1EndingPos; i++) {
        if (contains(results.deleted, i)) {
            deletedText += '<span class="deleted">';
        }
        deletedText += escapeHtml(text1Lines[i]).replace(/\t/g, '   ');
        if (contains(results.deleted, i)) {
            deletedText += '</span>';
        }
        deletedText += '<br>';
    }

    var addedText = '';
    for(i = startingPos; i < text2EndingPos; i++) {
        if (contains(results.added, i)) {
            addedText += '<span class="inserted">';
        }
        addedText += escapeHtml(text2Lines[i]).replace(/\t/g, '   ');
        if (contains(results.added, i)) {
            addedText += '</span>';
        }
        addedText += "<br>";
    }

    document.getElementById('original_result').innerHTML = deletedText;
    document.getElementById('edited_result').innerHTML = addedText;
}

function escapeHtml(string) {
    var entityMap = {
        "&": "&amp;",
        "<": "&lt;",
        ">": "&gt;",
        '"': '&quot;',
        "'": '&#39;',
        "/": '&#x2F;'
    };

    return String(string).replace(/[&<>"'\/]/g, function (s) {
        return entityMap[s];
    });
}
{'fraction_non_alphanumeric': 0.15881202205228526, 'fraction_numerical': 0.016361372932598258, 'mean_word_length': 2.4022988505747125, 'pattern_counts': {'":': 5, '<': 29, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3342683', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1775, 'n_words': 374}