<commit_before>/*!
 * coo-coo v0.0.2, https://github.com/hoho/coocoo
 * (c) 2013 <NAME>, MIT license
 */
var fs = require('fs'),
    util = require('./util.js'),
    extend = require('deep-extend');


var INDENT_WITH = ' ',
    INDENT = INDENT_WITH + INDENT_WITH + INDENT_WITH + INDENT_WITH,

    COO_COMMAND_PART_STRING = 'string',
    COO_COMMAND_PART_JS = 'JavaScript',
    COO_COMMAND_PART_IDENTIFIER = 'identifier',
    COO_COMMAND_PART_PROPERTY_GETTER = 'property getter',
    COO_COMMAND_PART_VARIABLE_GETTER = 'variable getter',
    COO_COMMAND_PART_TYPIFICATION = 'typification',
    COO_COMMAND_PART_SUBCOOCOO = 'sub CooCoo',

    COO_INTERNAL_VARIABLE_RET = '__ret';


function CooCommand(file, parent, decls) {
    this.debug = file.ret.debug;
    this.file = file;
    this.decls = decls;

    if (parent) {
        this.root = parent.root || parent;
    }

    this.parent = parent;
    this.children = [];
    this.data = {};
}

CooCommand.prototype = {
    root: null,
    valuePusher: false,
    valueRequired: false,
    noScope: false,
    hasSubblock: false,
    hasRet: false,
    parts: null,
    processChild: null,

    indent: 1,
    first: false,
    last: false,

    ignore: false,

    getDeclKey: null,

    getCodeBefore: null,
    getCodeAfter: null
};


function CooCommandPart(type, lineAt, charAt) {
    this.type = type;
    this._lineAt = lineAt;
    this._charAt = charAt;
}

CooCommandPart.prototype = {
    type: null,
    value: null,
    error: null
};


/* exported cooMatchCommand */
function cooMatchCommand(cmd, patterns, pos) {
    var parts = cmd.parts,
        part = parts[(pos = pos || 0)],
        error,
        unexpected = true;

    /*

     `patterns` is an object like:
     {
     something: callback,
     something: {something: callback},
     something: {something: {something: callback}}
     something: ...,
     ...
     }

     Where `something`:
     '' in case of any identifier,
     'Identifier' in case of exact identifier,
     '"' in case of any string,
     '"string"' in case of exact string,
     '(' in case of JavaScript,
     '*' in case of any number of identifiers,
     '#' in case of any number of strings or JavaScripts,
     '@' in case of callback.

     And `callback` is a callback to call when pattern is matched.

     Adding '<' in the end of somethings above will allow this something to be
     followed by typification.

     */

    if (part) {
        switch (part.type) {
            case COO_COMMAND_PART_STRING:
            case COO_COMMAND_PART_JS:
            case COO_COMMAND_PART_PROPERTY_GETTER:
            case COO_COMMAND_PART_VARIABLE_GETTER:
            case COO_COMMAND_PART_SUBCOOCOO:
                if (part.type === COO_COMMAND_PART_STRING) {
                    /* jshint -W061 */
                    var val = JSON.stringify(eval(part.value));
                    /* jshint +W061 */

                    if (patterns[val]) {
                        error = cooMatchCommand(cmd, patterns[val], pos + 1);
                        unexpected = false;
                    }
                }

                if ((error || unexpected) && patterns['(']) {
                    error = cooMatchCommand(cmd, patterns['('], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['(<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['(<'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#']) {
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#']) {
                    error = cooMatchCommand(cmd, patterns['#'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['#<'], pos + 1);
                    unexpected = false;
                }

                return unexpected ? part : error;

            case COO_COMMAND_PART_IDENTIFIER:
                if (patterns[part.value]) {
                    error = cooMatchCommand(cmd, patterns[part.value], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['']) {
                    error = cooMatchCommand(cmd, patterns[''], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['<'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['*']) {
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['*<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                return unexpected ? part : error;

            case COO_COMMAND_PART_TYPIFICATION:
                if (parts[pos - 1].typified) {
                    parts[pos - 1].typification = part;
                    parts.splice(pos, 1);
                    error = cooMatchCommand(cmd, patterns, pos);
                    unexpected = false;
                }

                return unexpected ? part : error;

            default:
                cmd.file.errorUnexpectedPart(part);
        }
    } else {
        switch ('function') {
            case typeof patterns['*']:
                return patterns['*'](cmd);<commit_msg>No multiple overriding typifications.<commit_after>/*!
 * coo-coo v0.0.2, https://github.com/hoho/coocoo
 * (c) 2013 <NAME>, MIT license
 */
var fs = require('fs'),
    util = require('./util.js'),
    extend = require('deep-extend');


var INDENT_WITH = ' ',
    INDENT = INDENT_WITH + INDENT_WITH + INDENT_WITH + INDENT_WITH,

    COO_COMMAND_PART_STRING = 'string',
    COO_COMMAND_PART_JS = 'JavaScript',
    COO_COMMAND_PART_IDENTIFIER = 'identifier',
    COO_COMMAND_PART_PROPERTY_GETTER = 'property getter',
    COO_COMMAND_PART_VARIABLE_GETTER = 'variable getter',
    COO_COMMAND_PART_TYPIFICATION = 'typification',
    COO_COMMAND_PART_SUBCOOCOO = 'sub CooCoo',

    COO_INTERNAL_VARIABLE_RET = '__ret';


function CooCommand(file, parent, decls) {
    this.debug = file.ret.debug;
    this.file = file;
    this.decls = decls;

    if (parent) {
        this.root = parent.root || parent;
    }

    this.parent = parent;
    this.children = [];
    this.data = {};
}

CooCommand.prototype = {
    root: null,
    valuePusher: false,
    valueRequired: false,
    noScope: false,
    hasSubblock: false,
    hasRet: false,
    parts: null,
    processChild: null,

    indent: 1,
    first: false,
    last: false,

    ignore: false,

    getDeclKey: null,

    getCodeBefore: null,
    getCodeAfter: null
};


function CooCommandPart(type, lineAt, charAt) {
    this.type = type;
    this._lineAt = lineAt;
    this._charAt = charAt;
}

CooCommandPart.prototype = {
    type: null,
    value: null,
    error: null
};


/* exported cooMatchCommand */
function cooMatchCommand(cmd, patterns, pos) {
    var parts = cmd.parts,
        part = parts[(pos = pos || 0)],
        error,
        unexpected = true;

    /*

     `patterns` is an object like:
     {
     something: callback,
     something: {something: callback},
     something: {something: {something: callback}}
     something: ...,
     ...
     }

     Where `something`:
     '' in case of any identifier,
     'Identifier' in case of exact identifier,
     '"' in case of any string,
     '"string"' in case of exact string,
     '(' in case of JavaScript,
     '*' in case of any number of identifiers,
     '#' in case of any number of strings or JavaScripts,
     '@' in case of callback.

     And `callback` is a callback to call when pattern is matched.

     Adding '<' in the end of somethings above will allow this something to be
     followed by typification.

     */

    if (part) {
        switch (part.type) {
            case COO_COMMAND_PART_STRING:
            case COO_COMMAND_PART_JS:
            case COO_COMMAND_PART_PROPERTY_GETTER:
            case COO_COMMAND_PART_VARIABLE_GETTER:
            case COO_COMMAND_PART_SUBCOOCOO:
                if (part.type === COO_COMMAND_PART_STRING) {
                    /* jshint -W061 */
                    var val = JSON.stringify(eval(part.value));
                    /* jshint +W061 */

                    if (patterns[val]) {
                        error = cooMatchCommand(cmd, patterns[val], pos + 1);
                        unexpected = false;
                    }
                }

                if ((error || unexpected) && patterns['(']) {
                    error = cooMatchCommand(cmd, patterns['('], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['(<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['(<'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#']) {
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#']) {
                    error = cooMatchCommand(cmd, patterns['#'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['#<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['#<'], pos + 1);
                    unexpected = false;
                }

                return unexpected ? part : error;

            case COO_COMMAND_PART_IDENTIFIER:
                if (patterns[part.value]) {
                    error = cooMatchCommand(cmd, patterns[part.value], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['']) {
                    error = cooMatchCommand(cmd, patterns[''], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns['<'], pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['*']) {
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                if ((error || unexpected) && patterns['*<']) {
                    part.typified = true;
                    error = cooMatchCommand(cmd, patterns, pos + 1);
                    unexpected = false;
                }

                return unexpected ? part : error;

            case COO_COMMAND_PART_TYPIFICATION:
                if (parts[pos - 1].typified && !parts[pos - 1].typification) {
                    parts[pos - 1].typification = part;
                    parts.splice(pos, 1);
                    error = cooMatchCommand(cmd, patterns, pos);
                    unexpected = false;
                }

                return unexpected ? part : error;

            default:
                cmd.file.errorUnexpectedPart(part);
        }
    } else {
        switch ('function') {
            case typeof patterns['*']:
                return patterns['*'](cmd);
{'fraction_non_alphanumeric': 0.11557707828894269, 'fraction_numerical': 0.004923325262308313, 'mean_word_length': 1.5595951249741788, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1940103', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3346, 'n_words': 838}