<commit_before>const fs = require('fs');
const path = require('path');

function getNextStep(content, pos, stop) {
    while (pos < content.length && content[pos] !== stop &&
           (content[pos] === ' ' || content[pos] === '\t' || content[pos] === '\n')) {
        pos += 1;
    }
    if (pos >= content.length) {
        return null;
    }
    if (content[pos] !== stop) {
        return pos * -1;
    }
    return pos;
}

// Stupid function extractor based on indent. Doesn't support block
// comments. If someone puts a ' or an " in a block comment this
// will blow up. Template strings are not tested and might also be
// broken.
function extractFunction(content, functionName) {
    var indent = 0;
    var splitter = "function " + functionName + "(";

    while (true) {
        var start = content.indexOf(splitter);
        if (start === -1) {
            break;
        }
        var pos = start;
        while (pos < content.length && content[pos] !== ')') {
            pos += 1;
        }
        if (pos >= content.length) {
            break;
        }
        pos = getNextStep(content, pos + 1, '{');
        if (pos === null) {
            break;
        } else if (pos < 0) {
            content = content.slice(-pos);
            continue;
        }
        while (pos < content.length) {
            // Eat single-line comments
            if (content[pos] === '/' && pos > 0 && content[pos-1] === '/') {
                do {
                    pos += 1;
                } while (pos < content.length && content[pos] !== '\n');

            // Eat quoted strings
            } else if (content[pos] === '"' || content[pos] === "'" || content[pos] === "`") {
                var stop = content[pos];
                var is_escaped = false;
                do {
                    if (content[pos] === '\\') {
                        pos += 2;
                    } else {
                        pos += 1;
                    }
                } while (pos < content.length &&
                         (content[pos] !== stop || content[pos - 1] === '\\'));

            // Otherwise, check for indent
            } else if (content[pos] === '{') {
                indent += 1;
            } else if (content[pos] === '}') {
                indent -= 1;
                if (indent === 0) {
                    return content.slice(start, pos + 1);
                }
            }
            pos += 1;
        }
        content = content.slice(start + 1);
    }
    return null;
}

// Stupid function extractor for array.
function extractArrayVariable(content, arrayName) {
    var splitter = "var " + arrayName;
    while (true) {
        var start = content.indexOf(splitter);
        if (start === -1) {
            break;
        }
        var pos = getNextStep(content, start, '=');
        if (pos === null) {
            break;
        } else if (pos < 0) {
            content = content.slice(-pos);<commit_msg>Fix rustdoc-js tool string "parsing" Improve tool: add support for multiline comments<commit_after>const fs = require('fs');
const path = require('path');

function getNextStep(content, pos, stop) {
    while (pos < content.length && content[pos] !== stop &&
           (content[pos] === ' ' || content[pos] === '\t' || content[pos] === '\n')) {
        pos += 1;
    }
    if (pos >= content.length) {
        return null;
    }
    if (content[pos] !== stop) {
        return pos * -1;
    }
    return pos;
}

// Stupid function extractor based on indent. Doesn't support block
// comments. If someone puts a ' or an " in a block comment this
// will blow up. Template strings are not tested and might also be
// broken.
function extractFunction(content, functionName) {
    var level = 0;
    var splitter = "function " + functionName + "(";
    var stop;
    var pos, start;

    while (true) {
        start = content.indexOf(splitter);
        if (start === -1) {
            break;
        }
        pos = start;
        while (pos < content.length && content[pos] !== ')') {
            pos += 1;
        }
        if (pos >= content.length) {
            break;
        }
        pos = getNextStep(content, pos + 1, '{');
        if (pos === null) {
            break;
        } else if (pos < 0) {
            content = content.slice(-pos);
            continue;
        }
        while (pos < content.length) {
            // Eat single-line comments
            if (content[pos] === '/' && pos > 0 && content[pos - 1] === '/') {
                do {
                    pos += 1;
                } while (pos < content.length && content[pos] !== '\n');

            // Eat multiline comment.
            } else if (content[pos] === '*' && pos > 0 && content[pos - 1] === '/') {
                do {
                    pos += 1;
                } while (pos < content.length && content[pos] !== '/' && content[pos - 1] !== '*');

            // Eat quoted strings
            } else if (content[pos] === '"' || content[pos] === "'" || content[pos] === "`") {
                stop = content[pos];
                do {
                    if (content[pos] === '\\') {
                        pos += 1;
                    }
                    pos += 1;
                } while (pos < content.length && content[pos] !== stop);

            // Otherwise, check for block level.
            } else if (content[pos] === '{') {
                level += 1;
            } else if (content[pos] === '}') {
                level -= 1;
                if (level === 0) {
                    return content.slice(start, pos + 1);
                }
            }
            pos += 1;
        }
        content = content.slice(start + 1);
    }
    return null;
}

// Stupid function extractor for array.
function extractArrayVariable(content, arrayName) {
    var splitter = "var " + arrayName;
    while (true) {
        var start = content.indexOf(splitter);
        if (start === -1) {
            break;
        }
        var pos = getNextStep(content, start, '=');
        if (pos === null) {
            break;
        } else if (pos < 0) {
            content = content.slice(-pos);
{'fraction_non_alphanumeric': 0.1558272208638957, 'fraction_numerical': 0.007334963325183374, 'mean_word_length': 1.398749022673964, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3721121', 'n_tokens_mistral': 1773, 'n_tokens_neox': 1674, 'n_words': 481}