<commit_before>'use strict';

import fs from 'fs';
import liner from './liner';
import randomizer from './randomizer';

function fillSmurfsArray(callback) {
    let array = [];
    let source = fs.createReadStream('./smurfs.txt');
    source.pipe(liner);

    liner.on('readable', function () {
        let line;
        while (line = liner.read()) {
            array.push({ name: line.trim() });
        }
    });

    liner.on('end', function() {
        callback(array);
    });
}

fillSmurfsArray(function(smurfsArray) {
    fs.writeFile('smurfs.json', JSON.stringify(smurfsArray), function(error) {
        if (error) console.log(error);
    });
});

console.log('Random smurf: ', randomizer.getRandomSmurf());<commit_msg>making reading file a little bit more generic<commit_after>'use strict';

import fs from 'fs';
import liner from './liner';
import randomizer from './randomizer';

function readFromFileLineByLine(filePath, readLineCallback, endReadingCallback) {
    let source = fs.createReadStream(filePath);
    source.pipe(liner);

    liner.on('readable', function () {
        let line;
        while (line = liner.read()) {
            readLineCallback(line);
        }
    });

    liner.on('end', function() {
        endReadingCallback(endReadingCallback);
    });
}

let newArray = [];
let filePath = './smurfs.txt';

function readLineCallback(line) {
    newArray.push({ name: line.trim() });
}

function endReadingCallback() {
    fs.writeFile('smurfs.json', JSON.stringify(newArray), function(error) {
        if (error) console.log(error);
    });
}

readFromFileLineByLine(filePath, readLineCallback, endReadingCallback);

console.log('Random smurf: ', randomizer.getRandomSmurf());
{'fraction_non_alphanumeric': 0.15389116442363956, 'fraction_numerical': 0.0, 'mean_word_length': 3.5, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1907478', 'n_tokens_mistral': 540, 'n_tokens_neox': 516, 'n_words': 108}