<commit_before>'use strict';
var cluster = require('cluster');
var fs = require('fs');
var os = require('os');
var Promise = require('bluebird');
var cheerio = require('cheerio');
var _ = require('lodash');
var request = require('superagent');

Promise.promisifyAll(request.Request.prototype);

if(cluster.isMaster) {
  setupWorkers(_.range(1000030, 1160639));
} else {
  process.on('message', function(grange) {
    worker(cluster.worker.id, grange);
  });
}

function setupWorkers(range) {
  var ncpus = os.cpus().length;
  var grange = groupBlocks(ncpus, range);

  for(var i = 0; i < ncpus; i++) {
    var worker = cluster.fork();
    worker.send(grange[i]);
  }
}

function groupBlocks(ncpus, range) {
  var blockSize = Math.floor(range.length / ncpus);
  var grange = [];

  for(var i = 0; i < ncpus; i++) {
    grange[i] = [];
    for(var j = 0; j < blockSize; j++) {
      grange[i].push(range[blockSize * i + j]);
    }
  }

  return grange;
}

function worker(id, range) {
  Promise.map(range, fetchPerson)
    .filter(function(a) { return a; })
    .then(function(entries) {
      console.log('Done!');
      fs.writeFileSync(
        id + '-people1.json',
        JSON.stringify(entries, null, 2)
      );
    });
}

function fetchPerson(id) {
  return request
    .get('http://www.fuvest.br/b/locexa2f.php')
    .query({
      s: id,
      anofuv: 2015
    })
    .endAsync()
    .then(function(res) {
      console.log('Finished downloading ' + id);
      return parseEntry(res.text);
    })
    .catch(function(/*err*/) {
      console.log('Failed downloading ' + id);
      return false;
    });
}

function parseEntry(html) {
  var $ = cheerio.load(html);
  var info = $('#idPageDIVContent div')
    .text()
    .split('\n')
    .slice(1, 5);

  if(/A pesquisa pelo texto abaixo n/.test(info[0])) {
    return false;
  }

  return {
    nome: info[0].split(' ')[1],
    inscricao: info[0].split(' ')[0],
    cpf: info[1].split(' ')[1],
    rg: info[2].split(' ')[1],
    convocadoParaSegundaFase: !/Você n/.test(info[3]),
  };
}
<commit_msg>Bug fixes and improvements<commit_after>'use strict';
var cluster = require('cluster');
var fs = require('fs');
var os = require('os');
var Promise = require('bluebird');
var cheerio = require('cheerio');
var _ = require('lodash');
var request = require('superagent');

Promise.promisifyAll(request.Request.prototype);

if(cluster.isMaster) {
  setupWorkers(_.range(1372161, 1372261));
} else {
  process.on('message', function(grange) {
    worker(cluster.worker.id, grange);
  });
}

function setupWorkers(range) {
  var ncpus = os.cpus().length;
  var grange = groupBlocks(ncpus, range);

  for(var i = 0; i < ncpus; i++) {
    var worker = cluster.fork();
    worker.send(grange[i]);
  }
}

function groupBlocks(ncpus, range) {
  var blockSize = Math.floor(range.length / ncpus);
  var grange = [];

  for(var i = 0; i < ncpus; i++) {
    grange[i] = [];
    for(var j = 0; j < blockSize; j++) {
      grange[i].push(range[blockSize * i + j]);
    }
  }

  return grange;
}

function worker(id, range) {
  Promise.map(range, fetchPerson)
    .filter(function(a) { return a; })
    .then(function(entries) {
      console.log('Done!');
      fs.writeFileSync(
        id + '-people1.json',
        JSON.stringify(entries, null, 2)
      );
    });
}

function fetchPerson(id) {
  return request
    .get('http://www.fuvest.br/b/locexa2f.php')
    .query({
      s: id,
      anofuv: 2015
    })
    .endAsync()
    .then(function(res) {
      console.log('Finished downloading ' + id);
      return parseEntry(res.text);
    })
    .catch(function(/*err*/) {
      console.log('Failed downloading ' + id);
      return false;
    });
}

function parseEntry(html) {
  var $ = cheerio.load(html);
  var info = $('#idPageDIVContent div')
    .text()
    .split('\n')
    .slice(1, 5);

  if(/A pesquisa pelo texto abaixo n/.test(info[0])) {
    return false;
  }

  return {
    nome: info[0].split(' ').slice(1).join(' '),
    inscricao: info[0].split(' ')[0],
    cpf: info[1].split(' ')[1],
    rg: info[2].split(' ')[1],
    convocadoParaSegundaFase: !/Você n/.test(info[3]),
  };
}

{'fraction_non_alphanumeric': 0.1881785283474065, 'fraction_numerical': 0.017370325693606754, 'mean_word_length': 3.044878048780488, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1954861', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1516, 'n_words': 307}