<commit_before>/*global parseFloat*/

/*
 * Thicket
 * https://github.com/d-plaindoux/thicket
 *
 * Copyright (c) 2015 <NAME>
 * Licensed under the LGPL2 license.
 */

module.exports = function () {

    'use strict';

    var parser = require('../../../Parser/parser.js'),
        bind = require('../../../Parser/bind.js'),
        eos = require('../../../Parser/eos.js'),
        opt = require('../../../Parser/opt.js'),
        optrep = require('../../../Parser/optrep.js'),
        rep = require('../../../Parser/rep.js'),
        choice = require('../../../Parser/choice.js'),
        commit = require('../../../Parser/commit.js'),
        ast  = require('./ast.js'),

        OPERATOR = /^([$#?;:@&!%><=+*/|.-]|\[|\])([$#?;:@&!%><=+*/|_.-]|\[|\])*/,
        SEPARATOR = [".", "(", ")", "{", "}", ",", "$", 
                     "<", ">", "/>", "</", "<-", "->", 
                     "=", "//", ":", ";;", "::", "$"],
        KEYWORDS = ["adapter", "module","from", "import", "typedef", "type", 
                    "model", "class", "def", "let", "in", "if", "for", "yield", 
                    "new", "with"],
        IDENT = /^[a-zA-Z_][a-zA-Z0-9_$]*/,
        ATTRIBUTE = /^[a-zA-Z_][a-zA-Z0-9_$-]*/,
        NUMBER = /^[+-]?\d+([.]\d*)?([eE][+-]?\d+)?/,
        HEXANUMBER = /0x[0-9a-fA-F]+/,
        DBQUOTEDSTRING = /^"[^"]*"/,
        QUOTEDSTRING = /^'[^']*'/,
        SPACES = /^\s+/,
        LINECOMMENT=/^\/\/[^\n]*\n?/,
        BLOCKCOMMENT=/^\/\*(.|\n)*?\*\//;

    function Language() {
        this.parser = parser();
            
        this.skip();
        this.ident();
        this.generics();

        this.module();
        this.source();
        this.moduleName();
        this.importations();
        this.entities();
        this.definition();
        this.adapter();
        this.types();
        this.model();
        this.controller();
        
        this.sentence();

        this.param();
        this.type();
        this.expression();
    }
    
    function locate(data,location) {
        function locateObject(data,location) {
            var key;
            
            for (key in data) {
                data[key] = locate(data[key], location);
            }
            
            return data;
        }
        
        if (!data) {
            return data;
        } else if (data.$location) {
            return data;
        } else if (data.$type) {
            return ast.locate(locateObject(data, location),location);
        } else {
            return data;
        }    
    }
    
    Language.prototype.locate = function () {
        this.parser.addLocationFn(locate);        
        return this;
    };

    Language.prototype.E = function (name) {
        if (this.parser.entry(name)) {
            return this.parser.entry(name);
        }

        throw new Error("Language entry named " + name + " not found");
    };

    //
    // Language rules definition
    //

    Language.prototype.skip = function () {
        // Define parse rules
        this.parser.addSkip(SPACES);
        this.parser.addSkip(LINECOMMENT);
        this.parser.addSkip(BLOCKCOMMENT);
    };

    Language.prototype.ident = function () {
        this.parser.group('ident').
            addRule(bind(IDENT).to('ident'), function (scope) {
                if (KEYWORDS.indexOf(scope.ident) > -1) {
                    return null;
                } else {
                    return scope.ident;
                }
            });
        this.parser.group('operator').
            addRule(bind(OPERATOR).to('ident'), function (scope) {
                if (SEPARATOR.indexOf(scope.ident) > -1) {
                    return null;
                } else {
                    return scope.ident;
                }
            });
    };

    Language.prototype.generics = function () {
        this.parser.group('generics').
            addRule(bind(opt(["[",rep(this.E("ident")),"]"])).to("generics"), function (scope) {
                if (scope.generics.length === 0) {
                    return [];
                } else {
                    return scope.generics[0][1];
                }
            });
    };
    
    Language.prototype.module = function() {
        // Source file -- Main entry
        this.parser.group("module").
            addRule(["module", bind(this.E("moduleNameDef")).to("namespace"), 
                     bind(this.E("source")).to("source")], function (scope) {
                return ast.module(scope.namespace, scope.source[0], scope.source[1], scope.source[2]);
            });
    };
    
    Language.prototype.source = function() {
        // Source toplevel -- Main entry
        this.parser.group("source").
            addRule([bind(this.E("importations")).to("importations"), 
                     bind(this.E("entities")).to("entities")], function (scope) {
                return [scope.importations, scope.entities[0], scope.entities[1]];
            });
    };
    
    Language.prototype.importations = function() {
        // importations
        this.parser.group("importations").
            addRule(bind(optrep(this.E("importationDef"))).to('importations'), function (scope) {
                return scope.importations;
            });
        
        this.parser.group("importationDef").
            addRule(["from", bind(this.E("moduleNameDef")).to("name"), 
                     "import", bind(this.E("importedItems")).to("names")], function (scope) {
                return ast.imports(scope.name, scope.names);
            });


        this.parser.group("importedItems").
            addRule("*", function () {
                return [];
            }).
            addRule(bind(rep(this.E("ident"))).to("idents"), function (scope) {
                return scope.idents;
            });
    };
    
    Language.prototype.moduleName = function () {
        this.parser.group("moduleNameDef").<commit_msg>Extend importation to operators<commit_after>/*global parseFloat*/

/*
 * Thicket
 * https://github.com/d-plaindoux/thicket
 *
 * Copyright (c) 2015 <NAME>
 * Licensed under the LGPL2 license.
 */

module.exports = function () {

    'use strict';

    var parser = require('../../../Parser/parser.js'),
        bind = require('../../../Parser/bind.js'),
        eos = require('../../../Parser/eos.js'),
        opt = require('../../../Parser/opt.js'),
        optrep = require('../../../Parser/optrep.js'),
        rep = require('../../../Parser/rep.js'),
        choice = require('../../../Parser/choice.js'),
        commit = require('../../../Parser/commit.js'),
        ast  = require('./ast.js'),

        OPERATOR = /^([$#?;:@&!%><=+*/|.-]|\[|\])([$#?;:@&!%><=+*/|_.-]|\[|\])*/,
        SEPARATOR = [".", "(", ")", "{", "}", ",", "$", 
                     "<", ">", "/>", "</", "<-", "->", 
                     "=", "//", ":", ";;", "::", "$"],
        KEYWORDS = ["adapter", "module","from", "import", "typedef", "type", 
                    "model", "class", "def", "let", "in", "if", "for", "yield", 
                    "new", "with"],
        IDENT = /^[a-zA-Z_][a-zA-Z0-9_$]*/,
        ATTRIBUTE = /^[a-zA-Z_][a-zA-Z0-9_$-]*/,
        NUMBER = /^[+-]?\d+([.]\d*)?([eE][+-]?\d+)?/,
        HEXANUMBER = /0x[0-9a-fA-F]+/,
        DBQUOTEDSTRING = /^"[^"]*"/,
        QUOTEDSTRING = /^'[^']*'/,
        SPACES = /^\s+/,
        LINECOMMENT=/^\/\/[^\n]*\n?/,
        BLOCKCOMMENT=/^\/\*(.|\n)*?\*\//;

    function Language() {
        this.parser = parser();
            
        this.skip();
        this.ident();
        this.generics();

        this.module();
        this.source();
        this.moduleName();
        this.importations();
        this.entities();
        this.definition();
        this.adapter();
        this.types();
        this.model();
        this.controller();
        
        this.sentence();

        this.param();
        this.type();
        this.expression();
    }
    
    function locate(data,location) {
        function locateObject(data,location) {
            var key;
            
            for (key in data) {
                data[key] = locate(data[key], location);
            }
            
            return data;
        }
        
        if (!data) {
            return data;
        } else if (data.$location) {
            return data;
        } else if (data.$type) {
            return ast.locate(locateObject(data, location),location);
        } else {
            return data;
        }    
    }
    
    Language.prototype.locate = function () {
        this.parser.addLocationFn(locate);        
        return this;
    };

    Language.prototype.E = function (name) {
        if (this.parser.entry(name)) {
            return this.parser.entry(name);
        }

        throw new Error("Language entry named " + name + " not found");
    };

    //
    // Language rules definition
    //

    Language.prototype.skip = function () {
        // Define parse rules
        this.parser.addSkip(SPACES);
        this.parser.addSkip(LINECOMMENT);
        this.parser.addSkip(BLOCKCOMMENT);
    };

    Language.prototype.ident = function () {
        this.parser.group('ident').
            addRule(bind(IDENT).to('ident'), function (scope) {
                if (KEYWORDS.indexOf(scope.ident) > -1) {
                    return null;
                } else {
                    return scope.ident;
                }
            });
        this.parser.group('operator').
            addRule(bind(OPERATOR).to('ident'), function (scope) {
                if (SEPARATOR.indexOf(scope.ident) > -1) {
                    return null;
                } else {
                    return scope.ident;
                }
            });
    };

    Language.prototype.generics = function () {
        this.parser.group('generics').
            addRule(bind(opt(["[",rep(this.E("ident")),"]"])).to("generics"), function (scope) {
                if (scope.generics.length === 0) {
                    return [];
                } else {
                    return scope.generics[0][1];
                }
            });
    };
    
    Language.prototype.module = function() {
        // Source file -- Main entry
        this.parser.group("module").
            addRule(["module", bind(this.E("moduleNameDef")).to("namespace"), 
                     bind(this.E("source")).to("source")], function (scope) {
                return ast.module(scope.namespace, scope.source[0], scope.source[1], scope.source[2]);
            });
    };
    
    Language.prototype.source = function() {
        // Source toplevel -- Main entry
        this.parser.group("source").
            addRule([bind(this.E("importations")).to("importations"), 
                     bind(this.E("entities")).to("entities")], function (scope) {
                return [scope.importations, scope.entities[0], scope.entities[1]];
            });
    };
    
    Language.prototype.importations = function() {
        // importations
        this.parser.group("importations").
            addRule(bind(optrep(this.E("importationDef"))).to('importations'), function (scope) {
                return scope.importations;
            });
        
        this.parser.group("importationDef").
            addRule(["from", bind(this.E("moduleNameDef")).to("name"), 
                     "import", bind(this.E("importedItems")).to("names")], function (scope) {
                return ast.imports(scope.name, scope.names);
            });


        this.parser.group("importedItems").
            addRule("*", function () {
                return [];
            }).
            addRule(bind(rep(this.E("methodName"))).to("idents"), function (scope) {
                return scope.idents;
            });
    };
    
    Language.prototype.moduleName = function () {
        this.parser.group("moduleNameDef").
{'fraction_non_alphanumeric': 0.1982271000422119, 'fraction_numerical': 0.0037146475306036302, 'mean_word_length': 1.5333618477331052, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5436563', 'n_tokens_mistral': 3559, 'n_tokens_neox': 3416, 'n_words': 512}