<commit_before>'use strict';

var express = require('express');
var bodyParser = require('body-parser');
var basicAuth = require('basic-auth-connect');
var http = require('http');
var cluster = require('cluster');
var redis = require('redis');
var url = require('url');

// Load configuration and initialize server
var config = require('./config/configuration.js');

var app = express();

app.use(bodyParser());

if(config.username && config.password) {
  app.use(basicAuth(config.username, config.password));
}

var components = url.parse(config.redisUrl);

var client = redis.createClient(components.port || 6379, components.hostname || "localhost", {
  auth_pass: (components.auth) ? ((components.auth.split(':').length > 1) ? components.auth.split(':')[1] : components.auth) : undefined
});

client.on('error', function(err) {
  console.warn('ERROR:', err);
});

var prefixesQueues = [];
var confQueues = [];

function refresh(cb) {
  client.keys("*:ids", function(err, keys) {
    if(err) {
      return cb(err);
    }

    var port = (confQueues.length === 0) ? ((config.slavePort) ? parseInt(config.slavePort) : parseInt(config.port) + 1) : confQueues[confQueues.length - 1].port + 1;

    keys.forEach(function(key) {
      key = key.replace(':ids', '');

      var matches = key.match(/^https?:\/\/([a-zA-Z0-9]+)/);

      if(!matches) {
        return;
      }

      if(prefixesQueues.indexOf(key) === -1) {
        confQueues.push({
          name: matches[1],
          prefix: key,
          port: port,
          launched: false
        });

        prefixesQueues.push(key);

        port += 1;
      }
    });

    confQueues.forEach(function(confQueue) {
      if(!confQueue.launched) {
        cluster.fork({
          QUEUE_PREFIX: confQueue.prefix,
          QUEUE_NAME: confQueue.name,
          QUEUE_PORT: confQueue.port
        });

        app.use('/' + confQueue.name, function(reqClient, resClient) {
          if(reqClient.url === "/") {
            reqClient.url = "/active";
          }

          http.get("http://localhost:" + confQueue.port + reqClient.url, function(resContent) {
            resClient.set(resContent.headers);
            resContent.pipe(resClient);
          });
        });
      }
    });

    cb(null);
  });
}

refresh(function(err) {
  if(err) {
    console.warn('ERROR:', err);
  }
});

app.engine('jade', require('jade').__express);
app.set('views', __dirname + "/views");
app.set('view engine', 'jade');

app.get('/', function(req, res) {
  res.render('index', {queues: confQueues});
});

app.get('/refresh', function(req, res) {
  refresh(function(err) {
    if(err) {<commit_msg>Set lauched to true after fork<commit_after>'use strict';

var express = require('express');
var bodyParser = require('body-parser');
var basicAuth = require('basic-auth-connect');
var http = require('http');
var cluster = require('cluster');
var redis = require('redis');
var url = require('url');

// Load configuration and initialize server
var config = require('./config/configuration.js');

var app = express();

app.use(bodyParser());

if(config.username && config.password) {
  app.use(basicAuth(config.username, config.password));
}

var components = url.parse(config.redisUrl);

var client = redis.createClient(components.port || 6379, components.hostname || "localhost", {
  auth_pass: (components.auth) ? ((components.auth.split(':').length > 1) ? components.auth.split(':')[1] : components.auth) : undefined
});

client.on('error', function(err) {
  console.warn('ERROR:', err);
});

var prefixesQueues = [];
var confQueues = [];

function refresh(cb) {
  client.keys("*:ids", function(err, keys) {
    if(err) {
      return cb(err);
    }

    var port = (confQueues.length === 0) ? ((config.slavePort) ? parseInt(config.slavePort) : parseInt(config.port) + 1) : confQueues[confQueues.length - 1].port + 1;

    keys.forEach(function(key) {
      key = key.replace(':ids', '');

      var matches = key.match(/^https?:\/\/([a-zA-Z0-9]+)/);

      if(!matches) {
        return;
      }

      if(prefixesQueues.indexOf(key) === -1) {
        confQueues.push({
          name: matches[1],
          prefix: key,
          port: port,
          launched: false
        });

        prefixesQueues.push(key);

        port += 1;
      }
    });

    confQueues.forEach(function(confQueue) {
      if(!confQueue.launched) {
        cluster.fork({
          QUEUE_PREFIX: confQueue.prefix,
          QUEUE_NAME: confQueue.name,
          QUEUE_PORT: confQueue.port
        });

        app.use('/' + confQueue.name, function(reqClient, resClient) {
          if(reqClient.url === "/") {
            reqClient.url = "/active";
          }

          http.get("http://localhost:" + confQueue.port + reqClient.url, function(resContent) {
            resClient.set(resContent.headers);
            resContent.pipe(resClient);
          });
        });

        confQueue.launched = true;
      }
    });

    cb(null);
  });
}

refresh(function(err) {
  if(err) {
    console.warn('ERROR:', err);
  }
});

app.engine('jade', require('jade').__express);
app.set('views', __dirname + "/views");
app.set('view engine', 'jade');

app.get('/', function(req, res) {
  res.render('index', {queues: confQueues});
});

app.get('/refresh', function(req, res) {
  refresh(function(err) {
    if(err) {
{'fraction_non_alphanumeric': 0.18343638748360502, 'fraction_numerical': 0.005621135469364812, 'mean_word_length': 2.9106227106227105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '837600', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1703, 'n_words': 307}