<commit_before>require('babel/register');
const {createReadStream, createWriteStream} = require('fs');

// model FileData
const FileData = require('./models/FileData.js');

// Initialize database as soon as possible
const mongoose = require('./infra/db-manager');
process.nextTick(() => {
    mongoose.connection.once('connected', () => {
        // drop db to start in a clean state
        console.log('Database intialized');
        mongoose.connection.db.dropDatabase();
    });
})

const readDictionaryStream =  createReadStream("app/fixtures/session_dictionary.txt");
const readSessionStream = createReadStream("app/fixtures/airbnb_session_data.txt");
// const readNodeJsEventLoopStream =  createReadStream("app/fixtures/nodejs_event_loop.txt");
const readNodeJsEventLoopStream =  createReadStream("app/fixtures/nodejs-event-loop-explanations.md");
const readReadmeStream =  createReadStream("README.md");
const writeLogStream = createWriteStream("app/log/log.txt");
// const readSessionStream = createReadStream("app/fixtures/session_data.txt");

let lines_ok=0, lines_ko=0;

// Read and save nodejs event loop explanation file
readNodeJsEventLoopStream.on("data", (data) => {
    const f = new FileData();
    f.value = data;
    f._id = "nodejs_event_loop";
    f.save( (err) => {
        if (err){
            console.log("ERR: "+err)        
        }
        else
            console.log("SUCCESS - NodeJs Event Loop explanations added");
    });
});
<commit_msg>Beginning of moving async tasks (read markdown files) to promises<commit_after>require('babel/register');
const {createReadStream, createWriteStream} = require('fs');

// model FileData
const FileData = require('./models/FileData.js');

// Initialize database as soon as possible
const mongoose = require('./infra/db-manager');
process.nextTick(() => {
    mongoose.connection.once('connected', () => {
        // drop db to start in a clean state
        console.log('Database intialized');
        mongoose.connection.db.dropDatabase();
    });
})


const readNodeJsEventLoopPath = "app/fixtures/nodejs-event-loop-explanations.md";

const readDictionaryStream =  createReadStream("app/fixtures/session_dictionary.txt");
const readSessionStream = createReadStream("app/fixtures/airbnb_session_data.txt");
// const readNodeJsEventLoopStream =  createReadStream("app/fixtures/nodejs_event_loop.txt");
const readNodeJsEventLoopStream =  createReadStream("app/fixtures/nodejs-event-loop-explanations.md");
const readReadmeStream =  createReadStream("README.md");
const writeLogStream = createWriteStream("app/log/log.txt");
// const readSessionStream = createReadStream("app/fixtures/session_data.txt");

let lines_ok=0, lines_ko=0;



const saveData = (data, id) => {
    const f = new FileData();
    f.value = data;
    f.id = id;
    // f._id = "nodejs_event_loop";
    f.save( (err) => {
        if (err){
            console.log("ERR: "+err)        
        }
        else
            console.log("SUCCESS - data added");
    });
}

const readNodeJsEventLoopFilePromise = new Promise( (saveData, reject) => {
    fs.readFile(nodeJsEventLoopPath, (err, data) => {
        err ? reject(err) : resolve(data)
    });
})

const readReadmeFilePromise = new Promise( (saveData, reject) => {
    fs.readFile(readMePath, (err, data) => {
        err ? reject(err) : resolve(data)
    });
})



const fs = require('fs');

const map = {
  'file1.txt': '111111111',
  'file2.txt': '222222222',
  'file3.txt': '333333333'
};

function writeFiles(files) {
  return Promise.all(Object.keys(files).map(fileName =>
    new Promise((resolve, reject) => {
      fs.writeFile(fileName, map[fileName], err => err ? reject(err) : resolve());
    })
  ));
}

writeFiles(map).then(() => console.log('Tada!'));


// Read and save nodejs event loop explanation file
readNodeJsEventLoopStream.on("data", (data) => {
    const f = new FileData();
    f.value = data;
    f._id = "nodejs_event_loop";
    f.save( (err) => {
        if (err){
            console.log("ERR: "+err)        
        }
        else
            console.log("SUCCESS - NodeJs Event Loop explanations added");
    });
});

{'fraction_non_alphanumeric': 0.14954172696575013, 'fraction_numerical': 0.008200675349734683, 'mean_word_length': 3.6026637069922307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2602445', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1274, 'n_words': 260}