<commit_before>var path = './';
var fs = require('fs');
var CronJob = require('cron').CronJob;
var utils = require('./utils.js');

//fs.watch(path, function () {
//    if (location)
//        location.reload();
//});

var Datastore = require('nedb')
    , db = new Datastore({ filename: 'openPDS.db', autoload: true });

var dropboxPath = "/Users/radugatej/Dropbox/Apps/";

exports.dataSourceParseJob = new CronJob('* */10 * * * *', function() {
    running = true;
    db.find({}, function (err, results) {
        results.forEach(function(dataSource) {
            var lastProcessingDate = dataSource.lastProcessingDate;
            if (!lastProcessingDate) {
                lastProcessingDate = new Date(0);
            }

            var filesToProcess = utils.getFilesAfterDate(dropboxPath + dataSource.dropboxFileLocation, lastProcessingDate);

            if (filesToProcess.length === 0) {
                return;
            }

            var dataTypeDBs = [];
            dataSource.dataTypes.forEach(function(dataType) {
                dataTypeDBs.push(dataType.cache_db_name);
            });
            db.update({_id: dataSource._id}, {$set: {lastProcessingDate: new Date()}}, {});

            utils.spawnPython(dataSource.parserPath, JSON.stringify({files: filesToProcess, dbs: dataTypeDBs}));
        });
    });
}, function () {

}, true, 'Europe/Copenhagen');

exports.db = db;

<commit_msg>Broken down dbs by type. Also temporarily removed cron scheduling<commit_after>var CronJob = require('cron').CronJob;
var utils = require('./utils.js');
var path = require('path');
var config = require('./config.js');
var dataSourceUtils = require('./data_sources.js');


var Datastore = require('nedb');
var db = {};


db.dataSources = new Datastore({filename: path.join(config.dataPath, 'openPDS_dataSources.db'), autoload: true});
db.dataSources.ensureIndex({fieldName: 'name', unique: true}, function(err) {
    //console.log(err.toString());
});

db.dataTypes = new Datastore({filename: path.join(config.dataPath, 'openPDS_dataTypes.db'), autoload: true});
db.dataTypes.ensureIndex({fieldName: 'name', unique: true}, function(err) {
    //console.log(err.toString());
});

db.answerModules = new Datastore({filename: path.join(config.dataPath, 'openPDS_answerModules.db'), autoload: true});
db.answerModules.ensureIndex({fieldName: 'name', unique: true}, function(err) {
    //console.log(err.toString());
});


//exports.dataSourceParseJob = new CronJob('* * * * * *', function () {
//    db.dataSources.find({}, function (err, results) {
//        results.forEach(function (dataSource) {
//            processRawDataSource(dataSource);
//        });
//    });
//}, function () {
//
//}, false, 'Europe/Copenhagen');

exports.db = db;


{'fraction_non_alphanumeric': 0.17271407837445574, 'fraction_numerical': 0.001451378809869376, 'mean_word_length': 3.3078125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5666558', 'n_tokens_mistral': 861, 'n_tokens_neox': 842, 'n_words': 162}