<commit_before>'use strict';
var not_whitespace_or_end = /^(\S|$)/;
var space_quote_paren_escaped_or_end = /^(\s|\\|"|'|\(|\)|$)/;
var string_or_escaped_or_end = /^(\\|"|$)/;
var quotes = /('|`|,)/;
var quotes_map = {
    '\'': 'quote',
    '`':  'quasiquote',
    ',':  'unquote'
};

function SParser(stream) {
    this._line = this._col = this._pos = 0;
    this._stream = stream;
}

SParser.prototype = {
    peek: peek,
    consume: consume,
    until: until,
    error: error,
    string: string,
    atom: atom,
    quoted: quoted,
    expr: expr,
    list: list
};

module.exports = function SParse(stream) {
    var parser = new SParser(stream);
    var expression = parser.expr();

    if (expression instanceof Error) {
        return expression;
    }

    // if anything is left to parse, it's a syntax error
    if (parser.peek() != '') {
        return parser.error('Superfluous characters after expression: `' + parser.peek() + '`');
    }

    return expression;
};

module.exports.SyntaxError = Error;

function error(msg) {
    var e = new Error('Syntax error: ' + msg);
    e.line = this._line + 1;
    e.col  = this._col + 1;
    return e;
}

function peek() {
    if (this._stream.length == this._pos) return '';
    return this._stream[this._pos];
}

function consume() {
    if (this._stream.length == this._pos) return '';

    var c = this._stream[this._pos];
    this._pos += 1;

    if (c == '\r') {
        if (this.peek() == '\n') {
            this._pos += 1;
            c += '\n';
        }
        this._line++;
        this._col = 0;
    } else if (c == '\n') {
        this._line++;
        this._col = 0;
    } else {
        this._col++;
    }

    return c;
}

function until(regex) {
    var s = '';

    while (!regex.test(this.peek())) {
        s += this.consume();
    }

    return s;
}

function string() {
    // consume "
    this.consume();

    var str = '';

    while (true) {
        str += this.until(string_or_escaped_or_end);
        var next = this.peek();

        if (next == '') {
            return this.error('Unterminated string literal');
        }

        if (next == '"') {
            this.consume();
            break;
        }

        if (next == '\\') {
            this.consume();
            next = this.peek();

            if (next == 'r') {
                this.consume();
                str += '\r';
            } else if (next == 't') {
                this.consume();
                str += '\t';
            } else if (next == 'n') {
                this.consume();
                str += '\n';
            } else if (next == 'f') {
                this.consume();
                str += '\f';
            } else if (next == 'b') {
                this.consume();
                str += '\b';
            } else {
                str += this.consume();
            }
        }
    }

    // wrap in object to make strings distinct from symbols
    return new String(str);
}

function atom() {
    if (this.peek() == '"') {
        return this.string();
    }

    var atom = '';

    while (true) {
        atom += this.until(space_quote_paren_escaped_or_end);
        var next = this.peek();

        if (next == '\\') {
            this.consume();
            atom += this.consume();
            continue;
        }

        break;
    }

    return atom;
}

function quoted() {
    var q = this.consume();
    var quote = quotes_map[q];

    if (quote == "unquote" && this.peek() == "@") {
        this.consume();
        quote = "unquote-splicing";
    }

    // ignore whitespace
    this.until(not_whitespace_or_end);
    var quotedExpr = this.expr();

    if (quotedExpr instanceof Error) {
        return quotedExpr;
    }

    // nothing came after '
    if (quotedExpr === '') {
        return this.error('Unexpected `' + this.peek() + '` after `' + q + '`');
    }

    if (quotedExpr instanceof Array && quotedExpr[0] !== quote) {
        quotedExpr.unshift(quote);
        return quotedExpr;
    }

    return [quote, quotedExpr];
}

function expr() {
    // ignore whitespace
    this.until(not_whitespace_or_end);

    if (quotes.test(this.peek())) {
        return this.quoted();
    }

    var expr = this.peek() == '(' ? this.list() : this.atom();

    // ignore whitespace
    this.until(not_whitespace_or_end);

    return expr;
}

function list() {
    if (this.peek() != '(') {
        return this.error('Expected `(` - saw `' + this.peek() + '` instead.');
    }<commit_msg>Implementation, passing quoted-list tests<commit_after>'use strict';
var not_whitespace_or_end = /^(\S|$)/;
var space_quote_paren_escaped_or_end = /^(\s|\\|"|'|\(|\)|$)/;
var string_or_escaped_or_end = /^(\\|"|$)/;
var quotes = /('|`|,)/;
var quotes_map = {
    '\'': 'quote',
    '`':  'quasiquote',
    ',':  'unquote'
};

function SParser(stream) {
    this._line = this._col = this._pos = 0;
    this._stream = stream;
}

SParser.prototype = {
    peek: peek,
    consume: consume,
    until: until,
    error: error,
    string: string,
    atom: atom,
    quoted: quoted,
    expr: expr,
    list: list
};

module.exports = function SParse(stream) {
    var parser = new SParser(stream);
    var expression = parser.expr();

    if (expression instanceof Error) {
        return expression;
    }

    // if anything is left to parse, it's a syntax error
    if (parser.peek() != '') {
        return parser.error('Superfluous characters after expression: `' + parser.peek() + '`');
    }

    return expression;
};

module.exports.SyntaxError = Error;

function error(msg) {
    var e = new Error('Syntax error: ' + msg);
    e.line = this._line + 1;
    e.col  = this._col + 1;
    return e;
}

function peek() {
    if (this._stream.length == this._pos) return '';
    return this._stream[this._pos];
}

function consume() {
    if (this._stream.length == this._pos) return '';

    var c = this._stream[this._pos];
    this._pos += 1;

    if (c == '\r') {
        if (this.peek() == '\n') {
            this._pos += 1;
            c += '\n';
        }
        this._line++;
        this._col = 0;
    } else if (c == '\n') {
        this._line++;
        this._col = 0;
    } else {
        this._col++;
    }

    return c;
}

function until(regex) {
    var s = '';

    while (!regex.test(this.peek())) {
        s += this.consume();
    }

    return s;
}

function string() {
    // consume "
    this.consume();

    var str = '';

    while (true) {
        str += this.until(string_or_escaped_or_end);
        var next = this.peek();

        if (next == '') {
            return this.error('Unterminated string literal');
        }

        if (next == '"') {
            this.consume();
            break;
        }

        if (next == '\\') {
            this.consume();
            next = this.peek();

            if (next == 'r') {
                this.consume();
                str += '\r';
            } else if (next == 't') {
                this.consume();
                str += '\t';
            } else if (next == 'n') {
                this.consume();
                str += '\n';
            } else if (next == 'f') {
                this.consume();
                str += '\f';
            } else if (next == 'b') {
                this.consume();
                str += '\b';
            } else {
                str += this.consume();
            }
        }
    }

    // wrap in object to make strings distinct from symbols
    return new String(str);
}

function atom() {
    if (this.peek() == '"') {
        return this.string();
    }

    var atom = '';

    while (true) {
        atom += this.until(space_quote_paren_escaped_or_end);
        var next = this.peek();

        if (next == '\\') {
            this.consume();
            atom += this.consume();
            continue;
        }

        break;
    }

    return atom;
}

function quoted() {
    var q = this.consume();
    var quote = quotes_map[q];

    if (quote == "unquote" && this.peek() == "@") {
        this.consume();
        quote = "unquote-splicing";
    }

    // ignore whitespace
    this.until(not_whitespace_or_end);
    var quotedExpr = this.expr();

    if (quotedExpr instanceof Error) {
        return quotedExpr;
    }

    // nothing came after '
    if (quotedExpr === '') {
        return this.error('Unexpected `' + this.peek() + '` after `' + q + '`');
    }

    return [quote, quotedExpr];
}

function expr() {
    // ignore whitespace
    this.until(not_whitespace_or_end);

    if (quotes.test(this.peek())) {
        return this.quoted();
    }

    var expr = this.peek() == '(' ? this.list() : this.atom();

    // ignore whitespace
    this.until(not_whitespace_or_end);

    return expr;
}

function list() {
    if (this.peek() != '(') {
        return this.error('Expected `(` - saw `' + this.peek() + '` instead.');
    }
{'fraction_non_alphanumeric': 0.17218393401875495, 'fraction_numerical': 0.0016947237600271157, 'mean_word_length': 1.7236923076923076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948200', 'n_tokens_mistral': 2959, 'n_tokens_neox': 2851, 'n_words': 652}