<commit_before>var _ = require('underscore');
var mongojs = require('mongojs');
var csv = require('./lib/csv');
var format = require('./lib/format');
var async = require('async');

var dbUrl = process.env.MONGOLAB_URI || 'mongodb://localhost/openerz',
    coll = ['calendar', 'station']
    db = mongojs(dbUrl, coll),

db.on('error',function(err) {
    console.log('database error', err);
});

db.on('ready',function() {
    console.log('db connected');
    // remove previous entries
    db.station.remove();
    db.calendar.remove();

    async.parallel([
        function(callback){
            csv.convertToJson('./csv/Entsorgung_Sammelstellen_2014.csv', format.stationEntry, function (objArr) {
                console.log('CSV converted, got ' + objArr.length + ' objects');

                _.each(objArr, function (obj) {
                    db.station.insert(obj);
                });
                console.log("Finished inserting stations!");
                callback(null, "stations");
            });
        },
        function(callback){
            csv.convertToJson('./csv/Entsorgungskalender_Bioabfall_2014.csv', format.calendarEntry, function (objArr) {
                console.log('CSV converted, got ' + objArr.length + ' objects');

                _.each(objArr, function (obj) {
                    obj['type'] = 'organic';
                    db.calendar.insert(obj);
                });
                console.log("Finished inserting calendar for organic waste!");<commit_msg>Load data in serie to avoid concurrency problems<commit_after>var _ = require('underscore');
var mongojs = require('mongojs');
var csv = require('./lib/csv');
var format = require('./lib/format');
var async = require('async');

var dbUrl = process.env.MONGOLAB_URI || 'mongodb://localhost/openerz',
    coll = ['calendar', 'station']
    db = mongojs(dbUrl, coll),

db.on('error',function(err) {
    console.log('database error', err);
});

db.on('ready', function() {
    console.log('db connected', dbUrl);
    // remove previous entries
    db.station.remove();
    db.calendar.remove();

    async.series([
        function(callback){
            csv.convertToJson('./csv/Entsorgung_Sammelstellen_2014.csv', format.stationEntry, function (objArr) {
                console.log('CSV converted, got ' + objArr.length + ' objects');

                _.each(objArr, function (obj) {
                    db.station.insert(obj);
                });
                console.log("Finished inserting stations!");
                callback(null, "stations");
            });
        },
        function(callback){
            csv.convertToJson('./csv/Entsorgungskalender_Bioabfall_2014.csv', format.calendarEntry, function (objArr) {
                console.log('CSV converted, got ' + objArr.length + ' objects');

                _.each(objArr, function (obj) {
                    obj['type'] = 'organic';
                    db.calendar.insert(obj);
                });
                console.log("Finished inserting calendar for organic waste!");
{'fraction_non_alphanumeric': 0.15119711380780584, 'fraction_numerical': 0.005247622171203673, 'mean_word_length': 2.3080260303687634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241767', 'n_tokens_mistral': 892, 'n_tokens_neox': 832, 'n_words': 164}