<commit_before>/**
 * Parsing the logs into a json object.
 */
const logger = require('winston');
const fs = require('fs');
const readLine = require('readline');
const Stream = require('stream');
const Database = require('../database');

const db = new Database();

/**
 *
 * @param data The data do be parsed.
 * @returns {Promise}
 */
function parseString (data) {
    logger.debug('Parsing logs as STRING');
    return new Promise((resolve, reject) => {
        let lines = [];
        let logEntry;
        data.toString().split('\n').forEach(function (line, idx, array) {
            if (line.length > 0) {
                if (idx !== array.length-1) {
                    logEntry = parseLine(line);
                    // filtering out not services
                    if (logEntry.service)
                        lines.push(logEntry);
                }
                else {
                    if (lines.length > 0) {
                        db.insert(lines);
                    }
                    return resolve(Buffer.byteLength(line,'utf-8'));
                }
            }
        })});
}

/**
 *
 * @param fileName Path to the file to be fully parsed.
 * @returns {Promise}
 */
function parseFile (fileName, start) {
    logger.debug('Parsing logs as FILE beggining at', start, 'bytes');
    return new Promise((resolve, reject) => {
        const inStream = fs.createReadStream(fileName, {});
        const outStream = new Stream;
        const rl = readLine.createInterface(inStream, outStream);
        let lines = [];
        let logEntry;
        let processed = 0;

        rl.on('line', function (line) {
            if (processed >= start) {
                if (line.length > 0) {
                    logEntry = parseLine(line);<commit_msg>fixes parsing issue<commit_after>/**
 * Parsing the logs into a json object.
 */
const logger = require('winston');
const fs = require('fs');
const readLine = require('readline');
const Stream = require('stream');
const Database = require('../database');

const db = new Database();

/**
 *
 * @param data The data do be parsed.
 * @returns {Promise}
 */
function parseString (data) {
    logger.debug('Parsing logs as STRING');
    return new Promise((resolve, reject) => {
        let lines = [];
        let logEntry;
        data.toString().split('\n').forEach(function (line, idx, array) {
            if (line.length > 0) {
                if (idx !== array.length-1) {
                    logEntry = parseLine(line);
                    // filtering out not services
                    if (logEntry.service !== 'NULL')
                        lines.push(logEntry);
                }
                else {
                    if (lines.length > 0) {
                        db.insert(lines);
                    }
                    return resolve(Buffer.byteLength(line,'utf-8'));
                }
            }
        })});
}

/**
 *
 * @param fileName Path to the file to be fully parsed.
 * @returns {Promise}
 */
function parseFile (fileName, start) {
    logger.debug('Parsing logs as FILE beggining at', start, 'bytes');
    return new Promise((resolve, reject) => {
        const inStream = fs.createReadStream(fileName, {});
        const outStream = new Stream;
        const rl = readLine.createInterface(inStream, outStream);
        let lines = [];
        let logEntry;
        let processed = 0;

        rl.on('line', function (line) {
            if (processed >= start) {
                if (line.length > 0) {
                    logEntry = parseLine(line);
{'fraction_non_alphanumeric': 0.131852684846781, 'fraction_numerical': 0.0033736294630306437, 'mean_word_length': 1.7560030983733539, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1859423', 'n_tokens_mistral': 968, 'n_tokens_neox': 926, 'n_words': 270}