<commit_before>var _        = require('underscore')
,   Epg      = require('./lib/epg')
,   conf     = require('./conf')
,   epg      = new Epg(conf.epg)
,   importer = require('./importer/' + conf.importer.type)(conf.importer.options)
,   status   = require('./status_client')('importer')
,   moment   = require('moment')
;

importer.on("error", function(err) {
    console.error("Error importing data", err);
//    status.set('status', 'error');
});
importer.on("services", function(services) {
    console.log("Received services from importer");
    epg.addServices(services);
});
importer.on("events", function(events) {
    console.log("Received events from importer");
    epg.addEvents(events);
    console.log("Updating EIT after import");
    var ret = epg.updateEit();
    if (ret) {
        console.log("EIT updated, updating Carousel");
        epg.updateCarousel();
    }
    importer.importFinished();
});
importer.on("run", function() {
    console.log("Starting import");
    status.set('status', 'running');
});
importer.on("done", function() {
    console.log("Import finished");
    status.set('status', 'idle');
    status.set('lastRun', moment().toISOString());
    status.set('nextRun', moment().add(importer.options.interval_hours, 'hours').toISOString());
});<commit_msg>After succesfull import, check memory and exit if it exceeds max configured<commit_after>var _        = require('underscore')
,   Epg      = require('./lib/epg')
,   conf     = require('./conf')
,   epg      = new Epg(conf.epg)
,   importer = require('./importer/' + conf.importer.type)(conf.importer.options)
,   status   = require('./status_client')('importer')
,   moment   = require('moment')
;

importer.on("error", function(err) {
    console.error("Error importing data", err);
//    status.set('status', 'error');
});
importer.on("services", function(services) {
    console.log("Received services from importer");
    epg.addServices(services);
});
importer.on("events", function(events) {
    console.log("Received events from importer");
    epg.addEvents(events);
    console.log("Updating EIT after import");
    var ret = epg.updateEit();
    if (ret) {
        console.log("EIT updated, updating Carousel");
        epg.updateCarousel();
    }
    importer.importFinished();
    var mem = process.memoryUsage();
    var memMB = mem.rss / 1048576;
    console.log("Memory used:", memMB, "MB");
    if (memMB > conf.importer.max_memory_mb) {
        console.log("Exceeded limit of", conf.importer.max_memory_mb, "MB, exiting...");
        process.exit();
    }
});
importer.on("run", function() {
    console.log("Starting import");
    status.set('status', 'running');
});
importer.on("done", function() {
    console.log("Import finished");
    status.set('status', 'idle');
    status.set('lastRun', moment().toISOString());
    status.set('nextRun', moment().add(importer.options.interval_hours, 'hours').toISOString());
});
{'fraction_non_alphanumeric': 0.18664850136239783, 'fraction_numerical': 0.0023841961852861036, 'mean_word_length': 3.767857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305834', 'n_tokens_mistral': 932, 'n_tokens_neox': 915, 'n_words': 172}