<commit_before>/*!
 * conkitty, https://github.com/hoho/conkitty
 * Copyright 2013-2014 <NAME>
 * Released under the MIT license
 */
'use strict';

var path = require('path'),

    ConkittyErrors = require(path.join(__dirname, 'errors.js')),
    ConkittyTypes = require(path.join(__dirname, 'types.js')),
    utils = require(path.join(__dirname, 'utils.js')),

    fs = require('fs'),

    whitespace = /[\x20\t\r\n\f]/,

    variableStopExpr = /[^a-zA-Z0-9_]/,
    variableCheckExpr = /^[a-zA-Z_][a-zA-Z0-9_]*$/,
    cssStopExpr = /[^a-zA-Z0-9_-]/,
    cssNameCheckExpr = /^[a-zA-Z][a-zA-Z0-9_-]*$/,
    tagCheckExpr = /^[a-z][a-zA-Z0-9_-]*$/,
    bemStopExpr = /[^a-zA-Z0-9-]/,
    bemCheckExpr = /^[a-zA-Z][a-zA-Z0-9-]*$/,

    commandExpr = /^(?:AS|ATTR|CALL|CHOOSE|EACH|ELSE|EXCEPT|EXPOSE|JS|MEM|OTHERWISE|PAYLOAD|SET|TEST|TRIGGER|WHEN|WITH)$/;


function clearComments(code) {
    var i,
        j,
        k,
        line,
        inComment,
        inString;

    i = 0;
    while (i < code.length) {
        line = code[i];

        if (!inComment) {
            inString = false;
            j = 0;

            while (j < line.length) {
                if (line[j] === "'" || line[j] === '"') {
                    if (inString === line[j] && line[j - 1] !== '\\') {
                        inString = false;
                        j++;
                        continue;
                    } else if (!inString) {
                        inString = line[j];
                        j++;
                        continue;
                    }
                }

                if (!inString) {
                    if (line[j] === '/' && (line[j + 1] === '/' || line[j + 1] === '*')) {
                        if (line[j + 1] === '*') {
                            k = line.indexOf('*/');

                            if (k > j + 1) {
                                line = line.substring(0, j) + new Array(k + 3 - j).join(' ') + line.substring(k + 2);
                                continue;
                            } else {
                                inComment = true;
                            }
                        }

                        line = line.substring(0, j);
                        break;
                    }
                }

                j++;
            }

            code[i] = line;
        } else { // In comment.
            k = line.indexOf('*/');

            if (k >= 0) {
                // Fill comment part with spaces.
                code[i] = new Array(k + 3).join(' ') + line.substring(k + 2);
                inComment = false;
                i--;
            } else {
                // Whole string is comment, clear it.
                code[i] = '';
            }
        }

        i++;
    }

    for (i = 0; i < code.length; i++) {
        code[i] = code[i].replace(/\s+$/g, '');
    }
}


function strip(str) {
    return str.replace(/^[\x20\t\r\n\f]+|[\x20\t\r\n\f]+$/g, '');
}


function ConkittyParser(filename, code, base, precompileEnv) {
    this.filename = filename;
    if (code === undefined) { code = fs.readFileSync(filename, {encoding: 'utf8'}); }
    this.base = base || path.dirname(filename);
    this.src = code.split(/\n\r|\r\n|\r|\n/);
    this.code = code.split(/\n\r|\r\n|\r|\n/);
    clearComments(this.code);
    this.lineAt = this.charAt = 0;
    this.chars = [];
    this.charsPos = 0;
    this.currentChar = {line: 0, col: 0};
    this.allowPrecompileExpr = true;
    this.precompileEnv = precompileEnv;
}


function ConkittyCommandPart(type, code, lineAt, charAt) {
    this.type = type;
    this.src = code;
    this.lineAt = lineAt === undefined ? code.currentChar.line : lineAt;
    this.charAt = charAt === undefined ? code.currentChar.col : charAt;
}


function execPrecompileExpr(expr, env) {
    var args = Object.keys(env || {}),
        i,
        values = [],
        func;

    for (i = 0; i < args.length; i++) {
        values.push(env[args[i]]);
    }

    func = new Function(
        args.join(','),
        'return ' + (expr.isFunc ? '(' + expr.value + ')()' : expr.value)
    );

    return func.apply(null, values) + '';
}


ConkittyParser.prototype.nextChar = function nextChar(noMove) {
    var ret;

    if (this.charsPos < this.chars.length) {
        ret = this.chars[this.charsPos];
        if (!noMove) {
            this.currentChar = ret;
            this.charsPos++;
        }
    } else {
        if (this.lineAt < this.code.length) {
            if (this.charAt < this.code[this.lineAt].length) {
                ret = {<commit_msg>Checking argument names.<commit_after>/*!
 * conkitty, https://github.com/hoho/conkitty
 * Copyright 2013-2014 <NAME>
 * Released under the MIT license
 */
'use strict';

var path = require('path'),

    ConkittyErrors = require(path.join(__dirname, 'errors.js')),
    ConkittyTypes = require(path.join(__dirname, 'types.js')),
    utils = require(path.join(__dirname, 'utils.js')),

    fs = require('fs'),

    whitespace = /[\x20\t\r\n\f]/,

    variableStopExpr = /[^a-zA-Z0-9_]/,
    variableCheckExpr = /^[a-zA-Z_][a-zA-Z0-9_]*$/,
    cssStopExpr = /[^a-zA-Z0-9_-]/,
    cssNameCheckExpr = /^[a-zA-Z][a-zA-Z0-9_-]*$/,
    tagCheckExpr = /^[a-z][a-zA-Z0-9_-]*$/,
    bemStopExpr = /[^a-zA-Z0-9-]/,
    bemCheckExpr = /^[a-zA-Z][a-zA-Z0-9-]*$/,

    commandExpr = /^(?:AS|ATTR|CALL|CHOOSE|EACH|ELSE|EXCEPT|EXPOSE|JS|MEM|OTHERWISE|PAYLOAD|SET|TEST|TRIGGER|WHEN|WITH)$/,

    argExpr = /^[a-zA-Z_$][0-9a-zA-Z_$]*$/;


function clearComments(code) {
    var i,
        j,
        k,
        line,
        inComment,
        inString;

    i = 0;
    while (i < code.length) {
        line = code[i];

        if (!inComment) {
            inString = false;
            j = 0;

            while (j < line.length) {
                if (line[j] === "'" || line[j] === '"') {
                    if (inString === line[j] && line[j - 1] !== '\\') {
                        inString = false;
                        j++;
                        continue;
                    } else if (!inString) {
                        inString = line[j];
                        j++;
                        continue;
                    }
                }

                if (!inString) {
                    if (line[j] === '/' && (line[j + 1] === '/' || line[j + 1] === '*')) {
                        if (line[j + 1] === '*') {
                            k = line.indexOf('*/');

                            if (k > j + 1) {
                                line = line.substring(0, j) + new Array(k + 3 - j).join(' ') + line.substring(k + 2);
                                continue;
                            } else {
                                inComment = true;
                            }
                        }

                        line = line.substring(0, j);
                        break;
                    }
                }

                j++;
            }

            code[i] = line;
        } else { // In comment.
            k = line.indexOf('*/');

            if (k >= 0) {
                // Fill comment part with spaces.
                code[i] = new Array(k + 3).join(' ') + line.substring(k + 2);
                inComment = false;
                i--;
            } else {
                // Whole string is comment, clear it.
                code[i] = '';
            }
        }

        i++;
    }

    for (i = 0; i < code.length; i++) {
        code[i] = code[i].replace(/\s+$/g, '');
    }
}


function strip(str) {
    return str.replace(/^[\x20\t\r\n\f]+|[\x20\t\r\n\f]+$/g, '');
}


function ConkittyParser(filename, code, base, precompileEnv) {
    this.filename = filename;
    if (code === undefined) { code = fs.readFileSync(filename, {encoding: 'utf8'}); }
    this.base = base || path.dirname(filename);
    this.src = code.split(/\n\r|\r\n|\r|\n/);
    this.code = code.split(/\n\r|\r\n|\r|\n/);
    clearComments(this.code);
    this.lineAt = this.charAt = 0;
    this.chars = [];
    this.charsPos = 0;
    this.currentChar = {line: 0, col: 0};
    this.allowPrecompileExpr = true;
    this.precompileEnv = precompileEnv;
}


function ConkittyCommandPart(type, code, lineAt, charAt) {
    this.type = type;
    this.src = code;
    this.lineAt = lineAt === undefined ? code.currentChar.line : lineAt;
    this.charAt = charAt === undefined ? code.currentChar.col : charAt;
}


function execPrecompileExpr(expr, env) {
    var args = Object.keys(env || {}),
        i,
        arg,
        values = [],
        func;

    for (i = 0; i < args.length; i++) {
        arg = args[i];
        if (!argExpr.test(arg)) {
            throw new Error('Invalid key `' + arg + '`');
        }
        values.push(env[arg]);
    }

    func = new Function(
        args.join(','),
        'return ' + (expr.isFunc ? '(' + expr.value + ')()' : expr.value)
    );

    return func.apply(null, values) + '';
}


ConkittyParser.prototype.nextChar = function nextChar(noMove) {
    var ret;

    if (this.charsPos < this.chars.length) {
        ret = this.chars[this.charsPos];
        if (!noMove) {
            this.currentChar = ret;
            this.charsPos++;
        }
    } else {
        if (this.lineAt < this.code.length) {
            if (this.charAt < this.code[this.lineAt].length) {
                ret = {
{'fraction_non_alphanumeric': 0.17531631996568733, 'fraction_numerical': 0.01072271070126528, 'mean_word_length': 1.3933795227097767, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '160493', 'n_tokens_mistral': 3228, 'n_tokens_neox': 3006, 'n_words': 564}