<commit_before>'use strict';

var DEBUG = true;
var kue = require('kue');

module.exports = function setup(options, imports, register) {
    console.log("Setup Core module.");
    var server = new Server(options, imports);
    if(server.start()){
      register(null, {
        "server": server
      });
    }else{
      console.log('error starting the server');
    }
};

var Server = function(options, imports){
    this.$scraper = imports.scraper;
    this.$resolveurl = imports.resolveurl;
    this.$db = imports.db;
    this.jobs = kue.createQueue({
      prefix: 'l',
      redis: {
        host : options.redis.host,
        port : options.redis.port,
        user : options.redis.username,
        pass : <PASSWORD>
      }
    });
};

(function(){

    this.start = function(data, done){
      var me = this;
      this.jobs.process('resolveUrl', function(job, done){
        me.doResolveUrl(job.data, done);
      });
      this.jobs.process('scrape', function(job, done){
        me.doScrapeUrl(job.data, done);
      });
      this.jobs.process('html', function(job, done){
        me.doScrapeHtml(job.data, done);
      });
      return true;
    };

    this.doResolveUrl = function (data, done) {
      DEBUG && console.log("doResolveUrl: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }

      this.$resolveurl.getOriginalURL(url, function(err, _url){
        if (err) {
          return done(new Error(err.message));
        }
        if (url != _url) {
          var result = {
            url: url,
            resolve: _url
          };
          console.log( _url );
          done(null, result);
        } else {
          done();
        }
      });
    };

    this.doScrapeUrl = function (data, done) {
      DEBUG && console.log("doScrapeUrl: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }
      this.$scraper.getStructuredData(url, done);
    };

    this.doScrapeHtml = function (data, done) {
      DEBUG && console.log("doScrapeHtml: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }

      this.$scraper.getLinksFromHtml(text, done);
    };

}).call(Server.prototype);<commit_msg>DEBUG is passed from app.js<commit_after>'use strict';

var kue = require('kue');

module.exports = function setup(options, imports, register) {
    console.log("Setup Core module.");
    var server = new Server(options, imports);
    if(server.start()){
      register(null, {
        "server": server
      });
    }else{
      console.log('error starting the server');
    }
};

var Server = function(options, imports){
    this.DEBUG = options.DEBUG,
    this.$scraper = imports.scraper;
    this.$resolveurl = imports.resolveurl;
    this.$db = imports.db;
    this.jobs = kue.createQueue({
      prefix: 'l',
      redis: {
        host : options.redis.host,
        port : options.redis.port,
        user : options.redis.username,
        pass : options.redis.password
      }
    });
};

(function(){

    this.start = function(data, done){
      var me = this;
      this.jobs.process('resolveUrl', function(job, done){
        me.doResolveUrl(job.data, done);
      });
      this.jobs.process('scrape', function(job, done){
        me.doScrapeUrl(job.data, done);
      });
      this.jobs.process('html', function(job, done){
        me.doScrapeHtml(job.data, done);
      });
      return true;
    };

    this.doResolveUrl = function (data, done) {
      this.DEBUG && console.log("doResolveUrl: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }
      this.$resolveurl.getOriginalURL(url, function(err, _url){
        if (err) {
          return done(new Error(err.message));
        }
        if (url != _url) {
          var result = {
            url: url,
            resolve: _url
          };
          done(null, result);
        } else {
          done();
        }
      });
    };

    this.doScrapeUrl = function (data, done) {
      this.DEBUG && console.log("doScrapeUrl: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }
      this.$scraper.getStructuredData(url, done);
    };

    this.doScrapeHtml = function (data, done) {
      this.DEBUG && console.log("doScrapeHtml: " + data.u);
      var url = data.u;
      if(!url){
        return done(new Error("Url required."));
      }
      this.$scraper.getLinksFromHtml(text, done);
    };

}).call(Server.prototype);
{'fraction_non_alphanumeric': 0.1551798872995232, 'fraction_numerical': 0.0, 'mean_word_length': 2.097315436241611, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '323291', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1451, 'n_words': 300}