<commit_before>/*
**  ComponentJS -- Component System for JavaScript <http://componentjs.com>
**  Copyright (c) 2009-2013 <NAME> <http://engelschall.com>
**
**  This Source Code Form is subject to the terms of the Mozilla Public
**  License, v. 2.0. If a copy of the MPL was not distributed with this
**  file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

(function () {

var pattern = /^[^<]*< ([^,]*), ([^,]*), ([^,]*), ([^,]*), ([^,]*), ([^,]*), (.*) >/

var tokens =  function () { return {
        data: [],
        add: function (token) {
            this.data.push(token)
        },
        len: function () {
            return this.data.length
        },
        first: function (offs) {
            if (offs === undefined) {
                return this.data[0]
            }
            return this.data[offs]
        },
        consume: function (symbol) {
            if (symbol !== this.data[0]) {
                if (symbol === '}' && this.data[0] === undefined)
                    throw new Error('parse error: we are missing a closing }')
                else
                    throw new Error('parse error: could not consume token "' + symbol +'", actual token "' + this.data[0] + '"')
            }
            this.data = this.data.slice(1, this.data.length)
        },
        skip: function () {
            this.data = this.data.slice(1, this.data.length)
        }
    }
}

var tokenize = function (src) {
    var tkns = new tokens()
    var match
    while(src !== '') {
        match = src.match(/^(\s*)(["'][^"']*["']|[^\[\]()):{}\/,\s]+|[():{}\[\]\/,])(\s*)/)
        if (match === null)
            throw new Error('parse error: "' + src + '"')
        tkns.add(match[2])
        src = src.substr(match[0].length)
    }

    return tkns
}

var parseObject = function (param, params) {
    while (params.first() !== '}') {
        if (params.first() === ',')
            params.skip()
        if (params.first(2) === '{') {
            var obj = {}
            param[params.first()] = obj
            params.skip()
            params.skip()
            params.skip()
            parseObject(obj, params)
        }
        else {
            param[params.first()] = JSON.parse(params.first(2))
            params.skip()
            params.skip()
            params.skip()
        }
    }
}

var parseLog = function (lines) {
    var tuples = []
    for (var i = 0; i < lines.length; i++) {
        var line = lines[i]
        var matches = line.match(pattern)
        var params = tokenize(matches[7])
        params.skip()
        var param = {}
        parseObject(param, params)
        var newTuple = {
            time: parseInt(matches[1], 10),
            source: matches[2],
            sourceType: matches[3],
            origin: matches[4],
            originType: matches[5],
            operation: matches[6],
            parameters: param
        }
        tuples.push(newTuple)
    }
    return tuples
}

app.lib.tupleParser = {}
app.lib.tupleParser.parseLog = parseLog

})()<commit_msg>Tuples are now sorted according to their time right after importing<commit_after>/*
**  ComponentJS -- Component System for JavaScript <http://componentjs.com>
**  Copyright (c) 2009-2013 <NAME> <http://engelschall.com>
**
**  This Source Code Form is subject to the terms of the Mozilla Public
**  License, v. 2.0. If a copy of the MPL was not distributed with this
**  file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

(function () {

var pattern = /^[^<]*< ([^,]*), ([^,]*), ([^,]*), ([^,]*), ([^,]*), ([^,]*), (.*) >/

var tokens =  function () { return {
        data: [],
        add: function (token) {
            this.data.push(token)
        },
        len: function () {
            return this.data.length
        },
        first: function (offs) {
            if (offs === undefined) {
                return this.data[0]
            }
            return this.data[offs]
        },
        consume: function (symbol) {
            if (symbol !== this.data[0]) {
                if (symbol === '}' && this.data[0] === undefined)
                    throw new Error('parse error: we are missing a closing }')
                else
                    throw new Error('parse error: could not consume token "' + symbol +'", actual token "' + this.data[0] + '"')
            }
            this.data = this.data.slice(1, this.data.length)
        },
        skip: function () {
            this.data = this.data.slice(1, this.data.length)
        }
    }
}

var tokenize = function (src) {
    var tkns = new tokens()
    var match
    while(src !== '') {
        match = src.match(/^(\s*)(["'][^"']*["']|[^\[\]()):{}\/,\s]+|[():{}\[\]\/,])(\s*)/)
        if (match === null)
            throw new Error('parse error: "' + src + '"')
        tkns.add(match[2])
        src = src.substr(match[0].length)
    }

    return tkns
}

var parseObject = function (param, params) {
    while (params.first() !== '}') {
        if (params.first() === ',')
            params.skip()
        if (params.first(2) === '{') {
            var obj = {}
            param[params.first()] = obj
            params.skip()
            params.skip()
            params.skip()
            parseObject(obj, params)
        }
        else {
            param[params.first()] = JSON.parse(params.first(2))
            params.skip()
            params.skip()
            params.skip()
        }
    }
}

var parseLog = function (lines) {
    var tuples = []
    for (var i = 0; i < lines.length; i++) {
        var line = lines[i]
        var matches = line.match(pattern)
        var params = tokenize(matches[7])
        params.skip()
        var param = {}
        parseObject(param, params)
        var newTuple = {
            time: parseInt(matches[1], 10),
            source: matches[2],
            sourceType: matches[3],
            origin: matches[4],
            originType: matches[5],
            operation: matches[6],
            parameters: param
        }
        tuples.push(newTuple)
    }

    tuples.sort(function (a, b) { return b.time - a.time })

    return tuples
}

app.lib.tupleParser = {}
app.lib.tupleParser.parseLog = parseLog

})()
{'fraction_non_alphanumeric': 0.1747901872175597, 'fraction_numerical': 0.010329244673983214, 'mean_word_length': 1.7037521815008727, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2243408', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1852, 'n_words': 446}