<commit_before>/**
 * Created by Jasper on 12.07.2017.
 */
"use strict";
const fs = require("fs")
const tokeniser = require("node-tokenizer")

tokeniser.rule("whitespace", /^[\r\n\t ]+/)
tokeniser.rule("arrayStart", /^[a-zA-Z]+\s*\[/)
tokeniser.rule("objectStart", /^[a-zA-Z]+\s*\{/)
tokeniser.rule("bracket", /^[\[\]\{\}]/)
tokeniser.rule("string", /^[^\r\n\[\]\{\}]+/)

let data = {}
let stack = [data]
let active = data
let toAdd = undefined

exports.readFile = (fileName) => {
    const lines = tokeniser.tokenize(fs.readFileSync(fileName).toString()).map(element => element.trim()).filter(element => !element.match(/^$/))
    data = {randomData: []}
    stack = [data]
    active = data
    toAdd = undefined
    lines.forEach(line => {
        //Test if comma
        if (line.substr(0, 2) === "//") return

        line = line.replace(/\\n/g, "\n")
        //Should add to last
        if (toAdd) {
            let cont = false
            if (line.substr(-1) === '\\') {
                line = line.substr(0, line.length - 1)
                cont = true
            }
            active[toAdd] += line.trim().replace(/^"|"$/g, '')
            if (!cont) {
                toAdd = undefined<commit_msg>Created preparser to clean lines for parsing in different functions<commit_after>/**
 * Created by Jasper on 12.07.2017.
 */
"use strict";
const fs = require("fs")
const tokeniser = require("node-tokenizer")

tokeniser.rule("whitespace", /^[\r\n\t ]+/)
tokeniser.rule("arrayStart", /^[a-zA-Z]+\s*\[/)
tokeniser.rule("objectStart", /^[a-zA-Z]+\s*\{/)
tokeniser.rule("bracket", /^[\[\]\{\}]/)
tokeniser.rule("string", /^[^\r\n\[\]\{\}]+/)

exports.preparseFile = (text) => {
    return tokeniser.tokenize(text).map(element => element.trim()).filter(element => !element.match(/^$/))
}

let data = {}
let stack = [data]
let active = data
let toAdd = undefined

exports.readFile = (fileName) => {
    const lines = preparseRile(fs.readFileSync(fileName).toString())
    data = {randomData: []}
    stack = [data]
    active = data
    toAdd = undefined
    lines.forEach(line => {
        //Test if comma
        if (line.substr(0, 2) === "//") return

        line = line.replace(/\\n/g, "\n")
        //Should add to last
        if (toAdd) {
            let cont = false
            if (line.substr(-1) === '\\') {
                line = line.substr(0, line.length - 1)
                cont = true
            }
            active[toAdd] += line.trim().replace(/^"|"$/g, '')
            if (!cont) {
                toAdd = undefined
{'fraction_non_alphanumeric': 0.21023622047244095, 'fraction_numerical': 0.010236220472440945, 'mean_word_length': 2.2660668380462723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3057498', 'n_tokens_mistral': 879, 'n_tokens_neox': 833, 'n_words': 178}