<commit_before>'use strict';

var init = require('../config/init')(),
    config = require('../config/config'),
    mongoose = require('mongoose');

var path = require('path');
var fs = require('fs');
var unorm = require('unorm');
var csv = require('csv');
var Q = require('q');

// Bootstrap db connection
var db = mongoose.connect(config.db, function(err) {
    if (err) {
        console.error(chalk.red('Could not connect to MongoDB!'));
        console.log(chalk.red(err));
    }
});

// Globbing model files
config.getGlobbedFiles('./app/models/**/*.js').forEach(function(modelPath) {
    require(path.resolve(modelPath));
});

var User = mongoose.model('User');
var Train = mongoose.model('Train');
var Track = mongoose.model('Track');
var Arrival = mongoose.model('Arrival');
var Departure = mongoose.model('Departure');

// parse csv files
var data = fs.readFileSync('./data/trains.csv');
csv.parse(data, function (err, items) {
    // items.shift();

    var arrivals = [];
    var tracks = [];
    var departures = [];
    var trains = [];

    items.forEach(function (item) {
        var train = {
            name: item[0],
            arrival: item[1],
            departure: item[2],
            arrivalTime: getTime(item[3]),
            departureTime: getTime(item[4]),
            track: item[5],
        };

        if (item[1] && arrivals.indexOf(item[1]) === -1) {
            arrivals.push(item[1]);
        }
        if (item[2] && departures.indexOf(item[2]) === -1) {
            departures.push(item[2]);
        }
        if (item[5] && tracks.indexOf(item[5]) === -1) {
            tracks.push(item[5]);
        }

        if (item[1] && item[2] && item[5]) {
            trains.push(train);
        }
    });

    var u;

    getUser()
        .then(function (user) {
            u = user;
            return addArrivals(arrivals, u);
        })
        .then(function (map) {
            trains.forEach(function (train) {
                train.arrival = map[train.arrival];
            });

            return addTracks(tracks, u);
        })
        .then(function (map) {
            trains.forEach(function (train) {
                train.track = map[train.track];
            });

            return addDepartures(departures, u);
        })
        .then(function (map) {
            trains.forEach(function (train) {
                train.departure = map[train.departure];
            });

            return addTrains(trains, u);
        })
        .then(function () {
            process.exit(0);
        });
});

function getUser() {
    var d = Q.defer();

    User.findOne({}, function (err, user) {
        if (err) { console.log('getUser:', err); }<commit_msg>update data import script<commit_after>'use strict';

var init = require('../config/init')(),
    config = require('../config/config'),
    mongoose = require('mongoose');

var path = require('path');
var fs = require('fs');
var unorm = require('unorm');
var csv = require('csv');
var Q = require('q');

// Bootstrap db connection
var db = mongoose.connect(config.db, function(err) {
    if (err) {
        console.error(chalk.red('Could not connect to MongoDB!'));
        console.log(chalk.red(err));
    }
});

// Globbing model files
config.getGlobbedFiles('./app/models/**/*.js').forEach(function(modelPath) {
    require(path.resolve(modelPath));
});

var User = mongoose.model('User');
var Train = mongoose.model('Train');
var Track = mongoose.model('Track');
var Arrival = mongoose.model('Arrival');
var Departure = mongoose.model('Departure');

function handleError(err, label, rowsDeleted) {
    if (err) {
        console.log('Error when truncate collection:', label);
        process.exit(1);
    } else {
        console.log('Deleted %d items from %s', rowsDeleted, label);
    }
}

Train.remove({}, function (err, rowsDeleted) {
    handleError(err, 'Train', rowsDeleted);
    Track.remove({}, function (err, rowsDeleted) {
        handleError(err, 'Track', rowsDeleted);
        Arrival.remove({}, function (err, rowsDeleted) {
            handleError(err, 'Arrival', rowsDeleted);
            Departure.remove({}, function (err, rowsDeleted) {
                handleError(err, 'Departure', rowsDeleted);
                if (!err) {
                    importData();
                }
            });
        });
    });
});

function importData() {
    // parse csv files
    var data = fs.readFileSync('./data/trains.csv');
    csv.parse(data, function (err, items) {
        // items.shift();

        var arrivals = [];
        var tracks = [];
        var departures = [];
        var trains = [];

        items.forEach(function (item) {
            var train = {
                name: item[0],
                arrival: item[1],
                departure: item[2],
                arrivalTime: getTime(item[3]),
                departureTime: getTime(item[4]),
                track: item[5],
            };

            if (item[1] && arrivals.indexOf(item[1]) === -1) {
                arrivals.push(item[1]);
            }
            if (item[2] && departures.indexOf(item[2]) === -1) {
                departures.push(item[2]);
            }
            if (item[5] && tracks.indexOf(item[5]) === -1) {
                tracks.push(item[5]);
            }

            if (item[1] && item[2] && item[5]) {
                trains.push(train);
            }
        });

        var u;

        getUser()
            .then(function (user) {
                u = user;
                return addArrivals(arrivals, u);
            })
            .then(function (map) {
                trains.forEach(function (train) {
                    train.arrival = map[train.arrival];
                });

                return addTracks(tracks, u);
            })
            .then(function (map) {
                trains.forEach(function (train) {
                    train.track = map[train.track];
                });

                return addDepartures(departures, u);
            })
            .then(function (map) {
                trains.forEach(function (train) {
                    train.departure = map[train.departure];
                });

                return addTrains(trains, u);
            })
            .then(function () {
                process.exit(0);
            });
    });
}

function getUser() {
    var d = Q.defer();

    User.findOne({}, function (err, user) {
        if (err) { console.log('getUser:', err); }
{'fraction_non_alphanumeric': 0.16344219161120568, 'fraction_numerical': 0.006964866119795697, 'mean_word_length': 1.8292469352014011, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1722415', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1935, 'n_words': 388}