<commit_before>'use strict';

const redisHost = process.env.REDIS_HOST || 'localhost',
  redisPort = process.env.REDIS_PORT || '6379';

const config = require('./config.json'),
  FromXml = require('xpath-generator').FromXml,
  Promise = require('bluebird'),
  redis = Promise.promisifyAll(require('redis')),
  fs = Promise.promisifyAll(require('fs')),
  path = require('path'),
  mkdirp = Promise.promisifyAll(require('mkdirp'));

config.xpathsOutput = config.xpathsOutput || '/applis/istex/job/';
config.redisDB = config.redisDB || config.redisDB;

const redisClient = redis.createClient(`//${redisHost}:${redisPort}`, {db: config.redisDB});

const sisypheXpath = {},
  xml = new FromXml();

const fullXpaths = new Set();

sisypheXpath.doTheJob = function (data, next) {
  if (data.mimetype !== 'application/xml' || !data.isWellFormed) {
    return next(null, data);
  }
  // console.log('Not Ignored: ', data.name);
  xml.generate(data.path, true).then(result => {
    //data.xpath = result;
    let keys = Object.keys(result);
    for (let i = 0; i < keys.length; i++) {
      redisClient.incrby(keys[i], result[keys[i]].count);
    }
    return next(null, data);
  }).catch(err => {
    return next(err);
  })
};

sisypheXpath.finalJob = function (done) {
  // When no more data in queue, sisyphe will execute it
  let outputPath = path.resolve(config.xpathsOutput, Date.now().toString()),
    outputFile = path.resolve(outputPath, 'xpaths-list.txt');

  mkdirp.mkdirpAsync(outputPath).then(() => {
    let xpathsStream = fs.createWriteStream(outputFile);
    xpathsStream.on('error', (err) => {
      return done(err);
    });
    xpathsStream.on('open', () => {
      scanAsync('0', '*').map((result) => {
        return xpathsStream.writeAsync(`${result.key} ${result.val}\n`);
      }).then(() => {
        xpathsStream.close();
        done();
      }).catch(err => {
        done(err)
      })
    });
    xpathsStream.on('close', () => {
      redisClient.flushdb();
    })
  }).catch(err => {
    return done(err);
  });
};

function scanAsync(cursor, pattern) {
  return redisClient.scanAsync(cursor, 'MATCH', pattern).then(reply => {
    cursor = +reply[0];
    for (let i = 0; i < reply[1].length; i++) {
      fullXpaths.add(reply[1][i]);
    }
    if (cursor !== 0) return scanAsync(cursor, '*');
    return Promise.map(fullXpaths, (key) => {
      return redisClient.getAsync(key).then(value => {
        return {key: key, val: value}
      });
    })
  });
}

module.exports = sisypheXpath;<commit_msg>Revert add xpath to data<commit_after>'use strict';

const redisHost = process.env.REDIS_HOST || 'localhost',
  redisPort = process.env.REDIS_PORT || '6379';

const config = require('./config.json'),
  FromXml = require('xpath-generator').FromXml,
  Promise = require('bluebird'),
  redis = Promise.promisifyAll(require('redis')),
  fs = Promise.promisifyAll(require('fs')),
  path = require('path'),
  mkdirp = Promise.promisifyAll(require('mkdirp'));

config.xpathsOutput = config.xpathsOutput || '/applis/istex/job/';
config.redisDB = config.redisDB || config.redisDB;

const redisClient = redis.createClient(`//${redisHost}:${redisPort}`, {db: config.redisDB});

const sisypheXpath = {},
  xml = new FromXml();

const fullXpaths = new Set();

sisypheXpath.doTheJob = function (data, next) {
  if (data.mimetype !== 'application/xml' || !data.isWellFormed) {
    return next(null, data);
  }
  xml.generate(data.path, true).then(result => {
    data.xpath = result;
    let keys = Object.keys(result);
    for (let i = 0; i < keys.length; i++) {
      redisClient.incrby(keys[i], result[keys[i]].count);
    }
    return next(null, data);
  }).catch(err => {
    return next(err);
  })
};

sisypheXpath.finalJob = function (done) {
  // When no more data in queue, sisyphe will execute it
  let outputPath = path.resolve(config.xpathsOutput, Date.now().toString()),
    outputFile = path.resolve(outputPath, 'xpaths-list.txt');

  mkdirp.mkdirpAsync(outputPath).then(() => {
    let xpathsStream = fs.createWriteStream(outputFile);
    xpathsStream.on('error', (err) => {
      return done(err);
    });
    xpathsStream.on('open', () => {
      scanAsync('0', '*').map((result) => {
        return xpathsStream.writeAsync(`${result.key} ${result.val}\n`);
      }).then(() => {
        xpathsStream.close();
        done();
      }).catch(err => {
        done(err)
      })
    });
    xpathsStream.on('close', () => {
      redisClient.flushdb();
    })
  }).catch(err => {
    return done(err);
  });
};

function scanAsync(cursor, pattern) {
  return redisClient.scanAsync(cursor, 'MATCH', pattern).then(reply => {
    cursor = +reply[0];
    for (let i = 0; i < reply[1].length; i++) {
      fullXpaths.add(reply[1][i]);
    }
    if (cursor !== 0) return scanAsync(cursor, '*');
    return Promise.map(fullXpaths, (key) => {
      return redisClient.getAsync(key).then(value => {
        return {key: key, val: value}
      });
    })
  });
}

module.exports = sisypheXpath;
{'fraction_non_alphanumeric': 0.188011988011988, 'fraction_numerical': 0.004395604395604396, 'mean_word_length': 3.4616755793226384, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7636251', 'n_tokens_mistral': 1743, 'n_tokens_neox': 1709, 'n_words': 316}