<commit_before>var delimiters = ['(', ')', ';', '\n'];
var whitespace = [' ', '\t', '\n'];

function makeStream(str) {
    return { pos: 0, string: str, len: str.length };
}

function peekChar(s) {
    if (s.pos < s.len)
        return s.string.charAt(s.pos);
}

function readChar(s) {
    var c = peekChar(s);

    if (c) {
        s.pos++;
        return c;
    }
}

function unreadChar(s) {
    s.pos > 0 && --s.pos;
}

function read(s) {
    readWhitespace(s);

    var c = peekChar(s);

    switch (c) {
        case ';': return readComment(s);
        case '(': return readList(s);
        case ')': throw new Error("Unexpected ) at " + s.pos);
        default: return readAtom(s);
    }
}

function readWhitespace (s) {
    var c;

    do {
        c = readChar(s);
    } while (c && whitespace.indexOf(c) >= 0);

    c && unreadChar(s);
}

function readAtom(s) {
    var c = readChar(s), str = '';

    while (c && whitespace.indexOf(c) < 0 && delimiters.indexOf(c) < 0) {
        str += c;
        c = readChar(s);
    }

    c && unreadChar(s);

    return str;
}

function readList(s) {
    readChar(s); // (

    var c, l = [];

    while ((c = peekChar(s)) && c != ')') {
        l.push(read(s));
    }

    if (c)
        readChar(s);
    else
        throw new Error('Expected ) at ' + s.pos);

    return l;
}

function readComment(s) {
    var c;
    do { c = readChar(s); } while (c && c != '\n');
}

function readFromString(str) {
    return read(makeStream(str));
}<commit_msg>Initial tests<commit_after>var delimiters = ['(', ')', ';', '\n'];
var whitespace = [' ', '\t', '\n'];

function makeStream(str) {
    return { pos: 0, string: str, len: str.length };
}

function peekChar(s) {
    if (s.pos < s.len)
        return s.string.charAt(s.pos);
}

function readChar(s) {
    var c = peekChar(s);

    if (c) {
        s.pos++;
        return c;
    }
}

function unreadChar(s) {
    s.pos > 0 && --s.pos;
}

function read(s) {
    readWhitespace(s);

    var c = peekChar(s);

    switch (c) {
        case ';': readComment(s); return read(s);
        case '(': return readList(s);
        case ')': throw new Error("Unexpected ) at " + s.pos);
        default: return readAtom(s);
    }
}

function readWhitespace (s) {
    var c;

    do {
        c = readChar(s);
    } while (c && whitespace.indexOf(c) >= 0);

    c && unreadChar(s);
}

function readAtom(s) {
    var c = readChar(s), str = '';

    while (c && whitespace.indexOf(c) < 0 && delimiters.indexOf(c) < 0) {
        str += c;
        c = readChar(s);
    }

    c && unreadChar(s);

    var n = parseFloat(str);

    return isNaN(n) ? str : n;
}

function readList(s) {
    readChar(s); // (

    var c, l = [];

    while ((c = peekChar(s)) && c != ')') {
        l.push(read(s));
    }

    if (c)
        readChar(s);
    else
        throw new Error('Expected ) at ' + s.pos);

    return l;
}

function readComment(s) {
    var c;
    do { c = readChar(s); } while (c && c != '\n');
}

function readFromString(str) {
    return read(makeStream(str));
}

function test(actual, expected) {
    if (expected !== actual) {
        throw new Error("Expected '" + expected + "', was '" + actual + "'");
    }
}

function runTests() {
    test(readFromString('()').length, [].length);
    test(readFromString('a'), 'a');
    test(readFromString('(a b c)').length, 3);
    test(readFromString('(a b c) ; blaz').length, 3);
    test(readFromString('; blaz\n(a b c)').length, 3);
    test(readFromString('( 1  b delta)').length, 3);
}
{'fraction_non_alphanumeric': 0.19545454545454546, 'fraction_numerical': 0.004261363636363636, 'mean_word_length': 2.287581699346405, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3530093', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1295, 'n_words': 308}