<commit_before>var less = exports || {};
//
// less.js - parser
//
//      A relatively straight-forward recursive-descent parser.
//      There is no tokenization/lexing stage, the input is parsed
//      in one sweep.
//
//      To make the parser fast enough to run in the browser, several
//      optimization had to be made:
//
//          - Instead of the more commonly used technique of slicing the
//          input string on every match, we use global regexps (/g),
//          and move the `lastIndex` pointer on match, foregoing `slice()`
//          completely. This gives us a 3x speed-up.
//
//          - Matching on a huge input is often cause of slowdowns,
//          especially with the /g flag. The solution to that is to
//          chunkify the input: we split it by /\n\n/, just to be on
//          the safe side. The chunks are stored in the `chunks` var,
//          `j` holds the current chunk index, and `current` holds
//          the index of the current chunk in relation to `input`.
//          This gives us an almost 4x speed-up.
//
//          - In many cases, we don't need to match individual tokens;
//          for example, if a value doesn't hold any variables, operations
//          or dynamic references, the parser can effectively 'skip' it,
//          treating it as a literal.
//          An example would be '1px solid #000' - which evaluates to itself,
//          we don't need to know what the individual components are.
//          The drawback, of course is that you don't get the benefits of
//          syntax-checking on the CSS. This gives us a 50% speed-up in the parser,
//          and a smaller speed-up in the code-gen.
//
//
//      Token matching is done with the `$` function, which either takes
//      a terminal string or regexp, or a non-terminal function to call.
//      It also takes care of moving all the indices forwards.
//
//
var input,       // LeSS input string
    i,           // current index in `input`
    j,           // current chunk
    chunks = [], // chunkified input
    current,     // index of current chunk, in `input`
    inputLength;

function peek(regex) {
    var match;
    regex.lastIndex = i;

    if ((match = regex.exec(input)) &&
        (regex.lastIndex - match[0].length === i)) {
        return match;
    }
}

//
// Parse from a token, regexp or string, and move forward if match
//
function $(tok, root) {
    var match, args, length, c, index;

    //
    // Non-terminal
    //
    if (tok instanceof Function) {
        return tok.call(less.parser.parsers, root);
    //
    // Terminal
    //
    //     Either match a single character in the input,
    //     or match a regexp in the current chunk (chunk[j]).
    //
    } else if (typeof(tok) === 'string') {
        match = input[i] === tok ? tok : null;
        length = 1;

    //  1. We move to the next chunk, if necessary.
    //  2. Set the `lastIndex` to be relative
    //     to the current chunk, and try to match in it.
    //  3. Make sure we matched at `index`. Because we use
    //     the /g flag, the match could be anywhere in the
    //     chunk. We have to make sure it's at our previous
    //     index, which we stored in [2].
    //
    } else {
        if (i > current + chunks[j].length) { // 1.
            current += chunks[j++].length;
        }
        tok.lastIndex = index =  i - current; // 2.
        match = tok.exec(chunks[j]);

        if (match) {
            length = match[0].length;
            if (tok.lastIndex - length !== index) { return } // 3.
        }
    }

    // The match is confirmed, add the match length to `i`,
    // and consume any extra white-space characters (' ' || '\n')
    // which come after that. The reason for this is that LeSS's
    // grammar is mostly white-space insensitive.
    //
    if (match) {
        i += length;

        while (i < inputLength) {
            c = input.charCodeAt(i);
            if (! (c === 32 || c === 10)) { break }
            i++;
        }
        return match.length === 1 ? match[0] : match;
    }
}

less.parser = {
    //
    // Parse an input string into an abstract syntax tree
    //
    parse: function (str) {
        var tree, start, end, zone, line;

        i = j = current = 0;
        input = str;
        inputLength = input.length;

        // Split the input into chunks,
        // delmited by /\n\n/ (see rationale above)
        // We use a lookahead, because we want to
        // preserve the '\n's in the input.
        chunks = input.split(/^(?=\n)/mg);

        // Start with the primary rule
        tree = new(node.Ruleset)([], $(this.parsers.primary, []));
        tree.root = true;

        // If `i` is smaller than the input length - 1,
        // it means the parser wasn't able to parse the whole
        // string, so we've got a parsing error.
        if (i < input.length - 1) {
            start = (function () {
                for (var n = i; n > 0; n--) {
                    if (input[n] === '\n') { break }
                }
                return n;
            })() + 1;
            line = input.slice(0, i).match(/\n/g).length + 1;
            end = input.slice(i).indexOf('\n') + i;
            zone = stylize(input.slice(start, i), 'green') +
                   stylize(input.slice(i, end), 'yellow');

            throw new(Error)("Parse Error on line " + line + ":\n" + zone);
        }
        return tree;
    },
    parsers: {
        primary: function primary(root) {
            var node;

            while (node = $(this.ruleset, []) || $(this.rule) || $(this.mixin.definition, []) ||
                          $(this.mixin.call)  || $(/\/\*([^*]|\*+[^\/*])*\*+\//g) || $(/[\n\s]+/g) || $(this.directive)) {
                root.push(node);
            }
            return root;
        },
        entities: {
            string: function string() {
                var str;
                if (input[i] !== '"' && input[i] !== "'") return;

                if (str = $(/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/g)) {<commit_msg>// comments<commit_after>var less = exports || {};
//
// less.js - parser
//
//      A relatively straight-forward recursive-descent parser.
//      There is no tokenization/lexing stage, the input is parsed
//      in one sweep.
//
//      To make the parser fast enough to run in the browser, several
//      optimization had to be made:
//
//          - Instead of the more commonly used technique of slicing the
//          input string on every match, we use global regexps (/g),
//          and move the `lastIndex` pointer on match, foregoing `slice()`
//          completely. This gives us a 3x speed-up.
//
//          - Matching on a huge input is often cause of slowdowns,
//          especially with the /g flag. The solution to that is to
//          chunkify the input: we split it by /\n\n/, just to be on
//          the safe side. The chunks are stored in the `chunks` var,
//          `j` holds the current chunk index, and `current` holds
//          the index of the current chunk in relation to `input`.
//          This gives us an almost 4x speed-up.
//
//          - In many cases, we don't need to match individual tokens;
//          for example, if a value doesn't hold any variables, operations
//          or dynamic references, the parser can effectively 'skip' it,
//          treating it as a literal.
//          An example would be '1px solid #000' - which evaluates to itself,
//          we don't need to know what the individual components are.
//          The drawback, of course is that you don't get the benefits of
//          syntax-checking on the CSS. This gives us a 50% speed-up in the parser,
//          and a smaller speed-up in the code-gen.
//
//
//      Token matching is done with the `$` function, which either takes
//      a terminal string or regexp, or a non-terminal function to call.
//      It also takes care of moving all the indices forwards.
//
//
var input,       // LeSS input string
    i,           // current index in `input`
    j,           // current chunk
    chunks = [], // chunkified input
    current,     // index of current chunk, in `input`
    inputLength;

function peek(regex) {
    var match;
    regex.lastIndex = i;

    if ((match = regex.exec(input)) &&
        (regex.lastIndex - match[0].length === i)) {
        return match;
    }
}

//
// Parse from a token, regexp or string, and move forward if match
//
function $(tok, root) {
    var match, args, length, c, index;

    //
    // Non-terminal
    //
    if (tok instanceof Function) {
        return tok.call(less.parser.parsers, root);
    //
    // Terminal
    //
    //     Either match a single character in the input,
    //     or match a regexp in the current chunk (chunk[j]).
    //
    } else if (typeof(tok) === 'string') {
        match = input[i] === tok ? tok : null;
        length = 1;

    //  1. We move to the next chunk, if necessary.
    //  2. Set the `lastIndex` to be relative
    //     to the current chunk, and try to match in it.
    //  3. Make sure we matched at `index`. Because we use
    //     the /g flag, the match could be anywhere in the
    //     chunk. We have to make sure it's at our previous
    //     index, which we stored in [2].
    //
    } else {
        if (i > current + chunks[j].length) { // 1.
            current += chunks[j++].length;
        }
        tok.lastIndex = index =  i - current; // 2.
        match = tok.exec(chunks[j]);

        if (match) {
            length = match[0].length;
            if (tok.lastIndex - length !== index) { return } // 3.
        }
    }

    // The match is confirmed, add the match length to `i`,
    // and consume any extra white-space characters (' ' || '\n')
    // which come after that. The reason for this is that LeSS's
    // grammar is mostly white-space insensitive.
    //
    if (match) {
        i += length;

        while (i < inputLength) {
            c = input.charCodeAt(i);
            if (! (c === 32 || c === 10)) { break }
            i++;
        }
        return match.length === 1 ? match[0] : match;
    }
}

less.parser = {
    //
    // Parse an input string into an abstract syntax tree
    //
    parse: function (str) {
        var tree, start, end, zone, line;

        i = j = current = 0;
        input = str;
        inputLength = input.length;

        // Split the input into chunks,
        // delmited by /\n\n/ (see rationale above)
        // We use a lookahead, because we want to
        // preserve the '\n's in the input.
        chunks = input.split(/^(?=\n)/mg);

        // Start with the primary rule
        tree = new(node.Ruleset)([], $(this.parsers.primary, []));
        tree.root = true;

        // If `i` is smaller than the input length - 1,
        // it means the parser wasn't able to parse the whole
        // string, so we've got a parsing error.
        if (i < input.length - 1) {
            start = (function () {
                for (var n = i; n > 0; n--) {
                    if (input[n] === '\n') { break }
                }
                return n;
            })() + 1;
            line = input.slice(0, i).match(/\n/g).length + 1;
            end = input.slice(i).indexOf('\n') + i;
            zone = stylize(input.slice(start, i), 'green') +
                   stylize(input.slice(i, end), 'yellow');

            throw new(Error)("Parse Error on line " + line + ":\n" + zone);
        }
        return tree;
    },
    parsers: {
        primary: function primary(root) {
            var node;

            while (node = $(this.ruleset, []) || $(this.rule) || $(this.mixin.definition, []) ||
                          $(this.mixin.call)  || $(/\/\*([^*]|\*+[^\/*])*\*+\//g) || $(/\/\/.*/g) ||
                          $(/[\n\s]+/g)       || $(this.directive)) {
                root.push(node);
            }
            return root;
        },
        entities: {
            string: function string() {
                var str;
                if (input[i] !== '"' && input[i] !== "'") return;

                if (str = $(/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/g)) {
{'fraction_non_alphanumeric': 0.14094345825115057, 'fraction_numerical': 0.005095332018408941, 'mean_word_length': 1.5842004671904863, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3664707', 'n_tokens_mistral': 3634, 'n_tokens_neox': 3488, 'n_words': 1332}