<commit_before>let Extender = require("./extender.js");
let TReader = require("./t-reader.js");
let Tools = require("./tools.js");

let GRE_WS = /\s+/g;

// array => array
function parse(tokens) {
    let tr = TReader.create(prepareTokens(tokens));
    let nodes = [];
    while (true) {
        if (tr.accept("$")) {
            return nodes;
        }
        if (tr.accept("text")) {
            nodes.push(parseText(tr));
        }
        if (tr.accept("tag_0")) {
            nodes.push(parseTag0(tr));
        }
        tr.expect("tag_1");
        nodes.push(parseTag1(tr));
    }
}

// array => array
function prepareTokens(tokens) {
    let result = [];
    let lastToken;
    tokens.forEach(token => {
        let newToken;
        if (token.type === "text") {
            let value = token.value.replace(GRE_WS, " ").trim();
            if (value !== "") {
                if (lastToken && lastToken.type === "text") {
                    lastToken.value += value;
                } else {
                    newToken = { type: "text", start: token.start, value };
                }
            }
        } else if (token.type !== "comment") {
            newToken = token;<commit_msg>Fix Parser<commit_after>let Extender = require("./extender.js");
let TReader = require("./t-reader.js");
let Tools = require("./tools.js");

let GRE_WS = /\s+/g;

// array => array
function parse(tokens) {
    let tr = TReader.create(prepareTokens(tokens));
    let nodes = [];
    while (true) {
        if (tr.accept("$")) {
            return nodes;
        }
        if (tr.accept("text")) {
            nodes.push(parseText(tr));
        } else if (tr.accept("tag_0")) {
            nodes.push(parseTag0(tr));
        } else {
            tr.expect("tag_1");
            nodes.push(parseTag1(tr));
        }
    }
}

// array => array
function prepareTokens(tokens) {
    let result = [];
    let lastToken;
    tokens.forEach(token => {
        let newToken;
        if (token.type === "text") {
            let value = token.value.replace(GRE_WS, " ").trim();
            if (value !== "") {
                if (lastToken && lastToken.type === "text") {
                    lastToken.value += value;
                } else {
                    newToken = { type: "text", start: token.start, value };
                }
            }
        } else if (token.type !== "comment") {
            newToken = token;
{'fraction_non_alphanumeric': 0.1682552838789888, 'fraction_numerical': 0.003315375051802735, 'mean_word_length': 1.664459161147903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7608034', 'n_tokens_mistral': 733, 'n_tokens_neox': 717, 'n_words': 150}