<commit_before>/*
 * loquat-token / token.js
 * copyright (c) 2016 Susisu
 */

/**
 * @module token
 */

"use strict";

module.exports = (_core, _prim, _char, _combinators) => {
    function end() {
        return Object.create({
            makeTokenParser
        });
    }

    const show         = _core.show;
    const unconsString = _core.unconsString;
    const lazy         = _core.lazy;
    const isParser     = _core.isParser;

    const map        = _prim.map;
    const pure       = _prim.pure;
    const bind       = _prim.bind;
    const then       = _prim.then;
    const ftailRecM  = _prim.ftailRecM;
    const mplus      = _prim.mplus;
    const label      = _prim.label;
    const unexpected = _prim.unexpected;
    const tryParse   = _prim.tryParse;
    const skipMany   = _prim.skipMany;
    const getConfig  = _prim.getConfig;

    const string     = _char.string;
    const satisfy    = _char.satisfy;
    const oneOf      = _char.oneOf;
    const noneOf     = _char.noneOf;
    const char       = _char.char;
    const space      = _char.space;
    const upper      = _char.upper;
    const digit      = _char.digit;
    const octDigit   = _char.octDigit;
    const hexDigit   = _char.hexDigit;
    const manyChars  = _char.manyChars;
    const manyChars1 = _char.manyChars1;

    const choice        = _combinators.choice;
    const option        = _combinators.option;
    const between       = _combinators.between;
    const many1         = _combinators.many1;
    const skipMany1     = _combinators.skipMany1;
    const sepBy         = _combinators.sepBy;
    const sepBy1        = _combinators.sepBy1;
    const notFollowedBy = _combinators.notFollowedBy;

    /*
     * white space
     */
    const spaceChars  = new Set(" \f\n\r\t\v");
    const simpleSpace = skipMany1(satisfy(char => spaceChars.has(char)));

    function oneLineComment(commentLine) {
        return then(
            tryParse(string(commentLine)),
            then(
                skipMany(satisfy(char => char !== "\n")),
                pure(undefined)
            )
        );
    }

    function multiLineComment(commentStart, commentEnd, nestedComments) {
        const commentStartEnd = commentStart + commentEnd;
        const inCommentMulti = lazy(() => label(
            mplus(
                then(
                    tryParse(string(commentEnd)),
                    pure(undefined)
                ),
                mplus(
                    then(comment, inCommentMulti),
                    mplus(
                        then(
                            skipMany1(noneOf(commentStartEnd)),
                            inCommentMulti
                        ),
                        then(
                            oneOf(commentStartEnd),
                            inCommentMulti
                        )
                    )
                )
            ),
            "end of comment"
        ));
        const inCommentSingle = lazy(() => label(
            mplus(
                then(
                    tryParse(string(commentEnd)),
                    pure(undefined)
                ),
                mplus(
                    then(
                        skipMany1(noneOf(commentStartEnd)),
                        inCommentSingle
                    ),
                    then(
                        oneOf(commentStartEnd),
                        inCommentSingle
                    )
                )
            ),
            "end of comment"
        ));
        const inComment = nestedComments ? inCommentMulti : inCommentSingle;
        const comment = then(
            tryParse(string(commentStart)),
            inComment
        );
        return comment;
    }

    /*
     * number literals
     */
    function number(base, baseDigit) {
        return bind(manyChars1(baseDigit), digits => pure(parseInt(digits, base)));
    }
    const decimal     = number(10, digit);<commit_msg>[token] Eliminate RangeError of `whiteSpace`<commit_after>/*
 * loquat-token / token.js
 * copyright (c) 2016 Susisu
 */

/**
 * @module token
 */

"use strict";

module.exports = (_core, _prim, _char, _combinators) => {
    function end() {
        return Object.create({
            makeTokenParser
        });
    }

    const show         = _core.show;
    const unconsString = _core.unconsString;
    const lazy         = _core.lazy;
    const isParser     = _core.isParser;

    const map        = _prim.map;
    const pure       = _prim.pure;
    const bind       = _prim.bind;
    const then       = _prim.then;
    const tailRecM   = _prim.tailRecM;
    const ftailRecM  = _prim.ftailRecM;
    const mplus      = _prim.mplus;
    const label      = _prim.label;
    const unexpected = _prim.unexpected;
    const tryParse   = _prim.tryParse;
    const skipMany   = _prim.skipMany;
    const getConfig  = _prim.getConfig;

    const string     = _char.string;
    const satisfy    = _char.satisfy;
    const oneOf      = _char.oneOf;
    const noneOf     = _char.noneOf;
    const char       = _char.char;
    const space      = _char.space;
    const upper      = _char.upper;
    const digit      = _char.digit;
    const octDigit   = _char.octDigit;
    const hexDigit   = _char.hexDigit;
    const manyChars  = _char.manyChars;
    const manyChars1 = _char.manyChars1;

    const choice        = _combinators.choice;
    const option        = _combinators.option;
    const between       = _combinators.between;
    const many1         = _combinators.many1;
    const skipMany1     = _combinators.skipMany1;
    const sepBy         = _combinators.sepBy;
    const sepBy1        = _combinators.sepBy1;
    const notFollowedBy = _combinators.notFollowedBy;

    /*
     * white space
     */
    const spaceChars  = new Set(" \f\n\r\t\v");
    const simpleSpace = skipMany1(satisfy(char => spaceChars.has(char)));

    function oneLineComment(commentLine) {
        return then(
            tryParse(string(commentLine)),
            then(
                skipMany(satisfy(char => char !== "\n")),
                pure(undefined)
            )
        );
    }

    function multiLineComment(commentStart, commentEnd, nestedComments) {
        const commentStartEnd = commentStart + commentEnd;
        const inCommentMulti = lazy(() =>
            tailRecM(
                undefined,
                () => label(
                    mplus(
                        then(
                            tryParse(string(commentEnd)),
                            pure({ done: true, value: undefined })
                        ),
                        mplus(
                            map(comment, () => ({ done: false, value: undefined })),
                            mplus(
                                map(
                                    skipMany1(noneOf(commentStartEnd)),
                                    () => ({ done: false, value: undefined })
                                ),
                                map(
                                    oneOf(commentStartEnd),
                                    () => ({ done: false, value: undefined })
                                )
                            )
                        )
                    ),
                    "end of comment"
                )
            )
        );
        const inCommentSingle = tailRecM(
            undefined,
            () => label(
                mplus(
                    then(
                        tryParse(string(commentEnd)),
                        pure({ done: true, value: undefined })
                    ),
                    mplus(
                        map(
                            skipMany1(noneOf(commentStartEnd)),
                            () => ({ done: false, value: undefined })
                        ),
                        map(
                            oneOf(commentStartEnd),
                            () => ({ done: false, value: undefined })
                        )
                    )
                ),
                "end of comment"
            )
        );
        const inComment = nestedComments ? inCommentMulti : inCommentSingle;
        const comment = then(
            tryParse(string(commentStart)),
            inComment
        );
        return comment;
    }

    /*
     * number literals
     */
    function number(base, baseDigit) {
        return bind(manyChars1(baseDigit), digits => pure(parseInt(digits, base)));
    }
    const decimal     = number(10, digit);
{'fraction_non_alphanumeric': 0.0897240164415737, 'fraction_numerical': 0.004227833235466823, 'mean_word_length': 1.249339672477549, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '650627', 'n_tokens_mistral': 2329, 'n_tokens_neox': 2197, 'n_words': 472}