<commit_before>/**
 * Worker that parses replays
 * The actual parsing is done by invoking the Java-based parser.
 * This produces an event stream (newline-delimited JSON)
 * Stream is run through a series of processors to count/aggregate it into a single object
 * This object is passed to insertMatch to persist the data into the database.
 * */
const utility = require('../util/utility');
const getGcData = require('../util/getGcData');
const config = require('../config');
const queue = require('../store/queue');
const queries = require('../store/queries');
// const createParsedDataBlob = require('../processors/createParsedDataBlob');
const cp = require('child_process');
const async = require('async');
const numCPUs = require('os').cpus().length;

const insertMatch = queries.insertMatch;
const buildReplayUrl = utility.buildReplayUrl;

function runParse(match, job, cb) {
  let url = match.url;
  if (config.NODE_ENV === 'test') {
    url = `https://cdn.rawgit.com/odota/testfiles/master/${match.match_id}_1.dem`;
  }
  console.log(new Date(), url);
  cp.exec(`curl --max-time 180 ${url} | ${url && url.slice(-3) === 'bz2' ? 'bunzip2' : 'cat'} | curl -X POST -T - ${config.PARSER_HOST} | node processors/createParsedDataBlob.js ${match.match_id} ${Boolean(match.doLogParse)}`,
    { shell: true, maxBuffer: 10 * 1024 * 1024 },
    (err, stdout) => {
      if (err) {
        return cb(err);
      }
      const result = Object.assign({}, JSON.parse(stdout), match);
      return insertMatch(result, {
        type: 'parsed',
        skipParse: true,
        doLogParse: match.doLogParse,
      }, cb);
    });
}

function parseProcessor(job, cb) {
  const match = job;
  async.series({
    getDataSource(cb) {
      getGcData(match, (err, result) => {
        if (err) {
          return cb(err);
        }
        match.url = buildReplayUrl(result.match_id, result.cluster, result.replay_salt);
        return cb(err);<commit_msg>add fail opt to curl<commit_after>/**
 * Worker that parses replays
 * The actual parsing is done by invoking the Java-based parser.
 * This produces an event stream (newline-delimited JSON)
 * Stream is run through a series of processors to count/aggregate it into a single object
 * This object is passed to insertMatch to persist the data into the database.
 * */
const utility = require('../util/utility');
const getGcData = require('../util/getGcData');
const config = require('../config');
const queue = require('../store/queue');
const queries = require('../store/queries');
// const createParsedDataBlob = require('../processors/createParsedDataBlob');
const cp = require('child_process');
const async = require('async');
const numCPUs = require('os').cpus().length;

const insertMatch = queries.insertMatch;
const buildReplayUrl = utility.buildReplayUrl;

function runParse(match, job, cb) {
  let url = match.url;
  if (config.NODE_ENV === 'test') {
    url = `https://cdn.rawgit.com/odota/testfiles/master/${match.match_id}_1.dem`;
  }
  console.log(new Date(), url);
  cp.exec(`curl --max-time 180 --fail ${url} | ${url && url.slice(-3) === 'bz2' ? 'bunzip2' : 'cat'} | curl -X POST -T - ${config.PARSER_HOST} | node processors/createParsedDataBlob.js ${match.match_id} ${Boolean(match.doLogParse)}`,
    { shell: true, maxBuffer: 10 * 1024 * 1024 },
    (err, stdout) => {
      if (err) {
        return cb(err);
      }
      const result = Object.assign({}, JSON.parse(stdout), match);
      return insertMatch(result, {
        type: 'parsed',
        skipParse: true,
        doLogParse: match.doLogParse,
      }, cb);
    });
}

function parseProcessor(job, cb) {
  const match = job;
  async.series({
    getDataSource(cb) {
      getGcData(match, (err, result) => {
        if (err) {
          return cb(err);
        }
        match.url = buildReplayUrl(result.match_id, result.cluster, result.replay_salt);
        return cb(err);
{'fraction_non_alphanumeric': 0.1540041067761807, 'fraction_numerical': 0.008726899383983572, 'mean_word_length': 3.3885135135135136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3164058', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1216, 'n_words': 330}