<commit_before>﻿// See https://github.com/myape/CodeMirror2/blob/0b109319eda1e12f9d1a7024c7be640eda21ae36/mode/sql/sql.js
CodeMirror.defineMode("sql", function (config, parserConfig) {
    var indentUnit = config.indentUnit,
        keywords = parserConfig.keywords,
        functions = parserConfig.functions,
        types = parserConfig.types,
        operators = parserConfig.operators,
        multiLineStrings = parserConfig.multiLineStrings;
    var isOperatorChar = /[+\-*&%=<>!?:\/|]/;

    function chain(stream, state, f) {
        state.tokenize = f;
        return f(stream, state);
    }

    var type;
    function ret(tp, style) {
        type = tp;
        return style;
    }

    function tokenBase(stream, state) {
        var ch = stream.next();
        // start of string?
        if (ch == "'") {
            return chain(stream, state, tokenString(ch));
        }
        // is it one of the special signs []{}().? 
        else if (/[\[\]{}\(\)\.]/.test(ch)) {
            return ret(ch);
        }
        // Seperator?
        else if (ch == "," || ch == ";") {
            return "sql-separator";
        }
        // start of a number value?
        else if (/\d/.test(ch)) {
            stream.eatWhile(/[\w\.]/)
            return ret("number", "sql-number");
        }
        // multi line comment or simple operator?
        else if (ch == "/") {
            if (stream.eat("*")) {
                return chain(stream, state, tokenComment);
            }
            else {<commit_msg>Fixed highlighter regression of http://meta.stackoverflow.com/q/51851<commit_after>﻿// See https://github.com/myape/CodeMirror2/blob/0b109319eda1e12f9d1a7024c7be640eda21ae36/mode/sql/sql.js
CodeMirror.defineMode("sql", function (config, parserConfig) {
    var indentUnit = config.indentUnit,
        keywords = parserConfig.keywords,
        functions = parserConfig.functions,
        types = parserConfig.types,
        operators = parserConfig.operators,
        multiLineStrings = parserConfig.multiLineStrings;
    var isOperatorChar = /[+\-*&%=<>!?:\/|]/;

    function chain(stream, state, f) {
        state.tokenize = f;
        return f(stream, state);
    }

    var type;
    function ret(tp, style) {
        type = tp;
        return style;
    }

    function tokenBase(stream, state) {
        var ch = stream.next();
        // start of string?
        if (ch == "'") {
            return chain(stream, state, tokenString(ch));
        }
        else if (ch == "[") {
            stream.skipTo("]");
            return "sql-word";
        }
        // is it one of the special signs {}().? 
        else if (/[{}\(\)\.]/.test(ch)) {
            return ret(ch);
        }
        // Seperator?
        else if (ch == "," || ch == ";") {
            return "sql-separator";
        }
        // start of a number value?
        else if (/\d/.test(ch)) {
            stream.eatWhile(/[\w\.]/)
            return ret("number", "sql-number");
        }
        // multi line comment or simple operator?
        else if (ch == "/") {
            if (stream.eat("*")) {
                return chain(stream, state, tokenComment);
            }
            else {
{'fraction_non_alphanumeric': 0.15324349733625822, 'fraction_numerical': 0.01723597618301473, 'mean_word_length': 1.7636363636363637, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2399937', 'n_tokens_mistral': 987, 'n_tokens_neox': 940, 'n_words': 214}