<commit_before>/**
 * Created by Alvin on 12/9/15.
 */

// 1. an almost complete asm grammar in simple JSON format
var asm_grammar = {

// prefix ID for regular expressions, represented as strings, used in the grammar
    "RegExpID"                  : "RE::",

    "Extra"                     : {

        "fold"                  : "indent"
    },

// Style model
    "Style"                     : {

        "comment"              : "comment"
        ,"decorator"            : "meta"
        ,"keyword"              : "keyword"
        ,"builtin"              : "builtin"
        ,"operator"             : "operator"
        ,"identifier"           : "variable"
        ,"number"               : "number"
        ,"string"               : "string"
        ,"heredoc"              : "string"

    },

// Lexical model
    "Lex"                       : {

        "comment:comment"      : ["#", null]
        ,"heredoc:block"        : [["'''"], ["\"\"\""], ["RE::/([rubRUB]|(ur)|(br)|(UR)|(BR))?('{3}|\"{3})/", 6]]
        ,"string:escaped-block" : [["RE::/(['\"])/", 1], ["RE::/([rubRUB]|(ur)|(br)|(UR)|(BR))?(['\"])/", 6]]
        ,"identifier"           : "RE::/[_A-Za-z][_A-Za-z0-9]*/"
        ,"decorator"            : "RE::/@[_A-Za-z][_A-Za-z0-9]*/"
        ,"number"               : [
            // floats
            "RE::/\\d*\\.\\d+(e[\\+\\-]?\\d+)?[jJ]?/",
            "RE::/\\d+\\.\\d*[jJ]?/",
            "RE::/\\.\\d+[jJ]?/",
            // integers
            // hex
            "RE::/0x[0-9a-fA-F]+[lL]?/",
            // binary
            "RE::/0b[01]+[lL]?/",
            // octal
            "RE::/0o[0-7]+[lL]?/",
            // decimal
            "RE::/[1-9]\\d*(e[\\+\\-]?\\d+)?[lL]?[jJ]?/",
            // just zero
            "RE::/0(?![\\dx])/"
        ]
        ,"operator"             : {"combine":false,"tokens":[
            "\\", "+", "-", "*", "/", "%", "&", "|", "^", "~", "<", ">" , "!",
            "==", "!=", "<=", ">=", "<>", "<<", ">>", "//", "**",
            "and", "or", "not", "is", "in"]}
        ,"delimiter"            : {"combine":false,"tokens":[
            "(", ")", "[", "]", "{", "}", ",", ":", "`", "=", ";", ".",
            "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
            ">>=", "<<=", "//=", "**=", "@"]}
        ,"keyword"              : {"autocomplete":true,"tokens":[
            "assert", "break", "class", "continue",
            "def", "del", "elif", "else", "except", "finally",
            "for", "from", "global", "if", "import",
            "lambda", "pass", "raise", "return",
            "try", "while", "with", "yield", "as"
        ]}
        ,"builtin"              : {"autocomplete":true,"tokens":[
            "abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr",
            "classmethod", "compile", "complex", "delattr", "dict", "dir", "divmod",
            "enumerate", "eval", "filter", "float", "format", "frozenset",
            "getattr", "globals", "hasattr", "hash", "help", "hex", "id",
            "input", "int", "isinstance", "issubclass", "iter", "len",
            "list", "locals", "map", "max", "memoryview", "min", "next",
            "object", "oct", "open", "ord", "pow", "property", "range",
            "repr", "reversed", "round", "set", "setattr", "slice",
            "sorted", "staticmethod", "str", "sum", "super", "tuple",
            "type", "vars", "zip", "__import__", "NotImplemented",
            "Ellipsis", "__debug__"
        ]}

    },

// Syntax model (optional)
    "Syntax"                    : {

        "asm"                    : "comment | heredoc | number | string | decorator | operator | delimiter | keyword | builtin | identifier"

    },

// what to parse and in what order
// an array i.e ["asm"], instead of single token i.e "asm", is a shorthand for an "ngram"-type syntax token (for parser use)
    "Parser"                    : [ ["asm"] ]

};

// 2. parse the grammar into a Codemirror syntax-highlight mode
var asm_mode = CodeMirrorGrammar.getMode( asm_grammar );


// 3. use it with Codemirror
CodeMirror.defineMode("asm", asm_mode);

// enable user-defined code folding in the specification (new feature)
asm_mode.supportCodeFolding = true;
CodeMirror.registerHelper("fold", asm_mode.foldType, asm_mode.folder);

// enable syntax lint-like validation in the grammar
asm_mode.supportGrammarAnnotations = true;
CodeMirror.registerHelper("lint", "asm", asm_mode.validator);

// enable user-defined autocompletion (if defined)
asm_mode.supportAutoCompletion = true;
CodeMirror.commands['my_autocompletion'] = function( cm ) {
    CodeMirror.showHint(cm, asm_mode.autocompleter, {prefixMatch:true, caseInsensitiveMatch:false});
};
// this also works (takes priority if set)
asm_mode.autocompleter.options = {prefixMatch:true, caseInsensitiveMatch:false};

<!-- TODO Create my own autocomplete for this ISA -->
var editor = CodeMirror.fromTextArea(document.getElementById("editor_box"), {
    //mode: "asm",
    lineNumbers: true,
    indentUnit: 4,
    indentWithTabs: false,<commit_msg>Have a buggy syntax highlighing working + autocompletion<commit_after>/**
 * Created by Alvin on 12/9/15.
 */

// TODO
// The things that my languages is composed of:
// - Labels, \.\w+\b
// - Comments, ;\w+\b
    // - Instructions, \b(SET|...)\b
    // - Registers, \b(SET|...)\b
// - Immediate values, \$\w+\b


// 1. an almost complete asm grammar in simple JSON format
var asm_grammar = {

// prefix ID for regular expressions, represented as strings, used in the grammar
    "RegExpID"                  : "RE::",

    "Extra"                     : {

        "fold"                  : "indent"
    },

// Style model
    "Style"                     : {

        "comment"               : "comment"
        ,"decorator"            : "meta"
        ,"Instruction"          : "keyword"
        ,"Registers"            : "builtin"
        ,"operator"             : "operator"
        ,"identifier"           : "variable"
        ,"number"               : "number"
        ,"label"               : "string"
    },

// Lexical model
    "Lex"                       : {

         "comment:comment"      : [";", null]
        ,"label"                : "RE::/[\.][_A-Za-z0-9]+/"
        ,"identifier"           : [
            // hex
            "RE::/[\$]0x[0-9a-fA-F]+/",
            // decimal
            "RE::/[\$][0-9]+/"
            ]
        ,"number"               : [
            // integers
            // hex
            "RE::/0x[0-9a-fA-F]+/",
            // decimal
            "RE::/[0-9]+/",
            // just zero
            "RE::/0(?![\\dx])/"
        ]
        ,"operator"             : {"combine":false,"tokens":[
            "\\", "+", "-", "*", "/", "%", "&", "|", "^", "~", "<", ">" , "!",
            "==", "!=", "<=", ">=", "<>", "<<", ">>", "//", "**",
            "and", "or", "not", "is", "in"]}
        ,"delimiter"            : {"combine":false,"tokens":[
            "(", ")", "[", "]", "{", "}", ",", ":", "`", "=", ";", ".",
            "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
            ">>=", "<<=", "//=", "**=", "@"]}
        ,"Instruction"          : {"autocomplete":true,"tokens":[
            "STP", "CCL", "PSH", "POP", "RTN", "JSR", "BRG", "BRZ", "BRA", "BRN", "CMP",
            "OR" , "AND", "LSH", "RSH", "DIV", "MUL", "SUB", "ADD", "MOV", "SET"
        ]}
        ,"Registers"            : {"autocomplete":true,"tokens":[
            "R0", "R1", "R2", "R3"
        ]}

    },

// Syntax model (optional)
    "Syntax"                    : {

        "asm"                    : "comment | number | label | decorator | operator | delimiter | Instruction | Registers | identifier"

    },

// what to parse and in what order
// an array i.e ["asm"], instead of single token i.e "asm", is a shorthand for an "ngram"-type syntax token (for parser use)
    "Parser"                    : [ ["asm"] ]

};

// 2. parse the grammar into a Codemirror syntax-highlight mode
var asm_mode = CodeMirrorGrammar.getMode( asm_grammar );


// 3. use it with Codemirror
CodeMirror.defineMode("asm", asm_mode);

// enable syntax lint-like validation in the grammar
asm_mode.supportGrammarAnnotations = true;
CodeMirror.registerHelper("lint", "asm", asm_mode.validator);

// enable user-defined autocompletion (if defined)
asm_mode.supportAutoCompletion = true;
CodeMirror.commands['my_autocompletion'] = function( cm ) {
    CodeMirror.showHint(cm, asm_mode.autocompleter, {prefixMatch:true, caseInsensitiveMatch:true});
};
// this also works (takes priority if set)
asm_mode.autocompleter.options = {prefixMatch:true, caseInsensitiveMatch:true};

<!-- TODO Create my own autocomplete for this ISA -->
var editor = CodeMirror.fromTextArea(document.getElementById("editor_box"), {
    //mode: "asm",
    lineNumbers: true,
    indentUnit: 4,
    indentWithTabs: false,
{'fraction_non_alphanumeric': 0.22653245871974667, 'fraction_numerical': 0.006333408731056322, 'mean_word_length': 1.8315722062119757, 'pattern_counts': {'":': 18, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1199157', 'n_tokens_mistral': 2987, 'n_tokens_neox': 2859, 'n_words': 585}