<commit_before>var fs       = require('fs')
,   zlib     = require('zlib')
,   readline = require('readline')
,   moment   = require('moment')
,   _        = require('underscore')
;

function data_provider(importDir, files, timezone) {
    this.importDir = importDir;
    this.files     = files;
    this._processTimezone(timezone);
}

data_provider.prototype._processTimezone = function(timezone) {
    if (!timezone) 
        return;
    var tmp = timezone.split(":");
    var hours = parseInt(tmp[0]);
    if (hours < 0) {
        hours = hours * -1;
        this.positiveTimezone = false;
    } else {
        this.positiveTimezone = true;
    }
    var minutes = parseInt(tmp[1]);
    var tz = moment.duration({hours: hours, minutes: minutes});
    this.timezone =  tz;
};

data_provider.prototype.init = function() {
    this._resetTables();
};

data_provider.prototype._resetTables = function() {
    this.services = [];
    this.programs = [];
    this.events   = [];
};

data_provider.prototype.processFile = function(file, callback) {
    var self = this;
    var gzip = zlib.createGunzip();
    var inp = fs.createReadStream(self.importDir + "/" + file);
    console.log("Unzipping", file);
    var out = inp.pipe(gzip)//.pipe(iconvlite.decodeStream('CP1250'))
            .on('end', function() {
                rl.close();
                console.log("End processing", file);
                if (file === self.files.services)
                    setTimeout(function() {callback({emit: true, type: "services", data: self.services});}, 3000);
                else if (file === self.files.programs)
                    callback({emit: false});
                else if (file === self.files.scheds)
                    setTimeout(function() {callback({emit: true, type: "events", data: self.events});}, 3000);
            });
    var rl = readline.createInterface({
        input: out,
        terminal: false
    });
    rl.on('line', function(line) {
        var data = line.split("|");
        if (file === self.files.services)
            self._processService(data);
        else if (file === self.files.programs)
            self._processProgram(data);
        else if (file === self.files.scheds)
            self._processSched(data);
    });
};

data_provider.prototype._processService = function(data) {
    var service = {};
    service.serviceId = parseInt(data[0]);<commit_msg>Found reportv enconding!!! (cp865)<commit_after>var fs        = require('fs')
,   zlib      = require('zlib')
,   readline  = require('readline')
,   moment    = require('moment')
,   _         = require('underscore')
,   iconvlite = require('iconv-lite')
;

function data_provider(importDir, files, timezone) {
    this.importDir = importDir;
    this.files     = files;
    this._processTimezone(timezone);
}

data_provider.prototype._processTimezone = function(timezone) {
    if (!timezone) 
        return;
    var tmp = timezone.split(":");
    var hours = parseInt(tmp[0]);
    if (hours < 0) {
        hours = hours * -1;
        this.positiveTimezone = false;
    } else {
        this.positiveTimezone = true;
    }
    var minutes = parseInt(tmp[1]);
    var tz = moment.duration({hours: hours, minutes: minutes});
    this.timezone =  tz;
};

data_provider.prototype.init = function() {
    this._resetTables();
};

data_provider.prototype._resetTables = function() {
    this.services = [];
    this.programs = [];
    this.events   = [];
};

data_provider.prototype.processFile = function(file, callback) {
    var self = this;
    var gzip = zlib.createGunzip();
    var inp = fs.createReadStream(self.importDir + "/" + file);
    console.log("Unzipping", file);
    var out = inp.pipe(gzip).pipe(iconvlite.decodeStream('cp865'))
            .on('end', function() {
                rl.close();
                console.log("End processing", file);
                if (file === self.files.services)
                    setTimeout(function() {callback({emit: true, type: "services", data: self.services});}, 3000);
                else if (file === self.files.programs)
                    callback({emit: false});
                else if (file === self.files.scheds)
                    setTimeout(function() {callback({emit: true, type: "events", data: self.events});}, 3000);
            });
    var rl = readline.createInterface({
        input: out,
        terminal: false
    });
    rl.on('line', function(line) {
        var data = line.split("|");
        if (file === self.files.services)
            self._processService(data);
        else if (file === self.files.programs)
            self._processProgram(data);
        else if (file === self.files.scheds)
            self._processSched(data);
    });
};

data_provider.prototype._processService = function(data) {
    var service = {};
    service.serviceId = parseInt(data[0]);
{'fraction_non_alphanumeric': 0.15207088398928498, 'fraction_numerical': 0.00741809190191634, 'mean_word_length': 2.4231311706629053, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6119126', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1429, 'n_words': 294}