<commit_before>'use strict';

/**
 * @file Helper for pdf2htmlEX file processing
 * For more information about Pdf2htmlex :
 * https://github.com/coolwanglu/pdf2htmlEX.git
 */

var fs = require('fs');
var async = require('async');
var shellExec = require('child_process').exec;
var crypto = require('crypto');

/**
 * Extract the content in html of the specified pdf
 *
 * @param {string} path Path of the specified file
 * @param {string} document to hydrate
 * @param {function} cb Callback, first parameter, is the error if any, then the processed data
 */
module.exports = function(path, document, cb) {

  async.waterfall([
    function(cb) {
      var outFile = crypto.randomBytes(20).toString('hex');

      shellExec('pdf2htmlEX --embed-javascript 0 --dest-dir /tmp ' + path + ' ' + outFile, function(err) {
        cb(err, '/tmp/' + outFile);
      });
    },
    function(outPath, cb) {
      fs.readFile(outPath, function(err, data) {
        cb(err, outPath, data);
      });
    },
    function(outPath, data, cb) {
      if(!document.metadatas) {
        document.metadatas = {};
      }

      document.metadatas.html = data.toString();
      document.binary_document_type = "pdf-html";
      cb(null, outPath);
    },
    function(outPath, cb) {
      fs.unlink(outPath, function() {
        cb();
      });
    },
  ], function(err) {
    cb(err, document);
  });
};<commit_msg>Cleanup<commit_after>'use strict';

/**
 * @file Helper for pdf2htmlEX file processing
 * For more information about Pdf2htmlex :
 * https://github.com/coolwanglu/pdf2htmlEX.git
 */

var fs = require('fs');
var async = require('async');
var shellExec = require('child_process').exec;
var crypto = require('crypto');

/**
 * Extract the content in html of the specified pdf
 *
 * @param {string} path Path of the specified file
 * @param {string} document to hydrate
 * @param {function} cb Callback, first parameter, is the error if any, then the processed data
 */
module.exports = function(path, document, cb) {

  async.waterfall([
    function(cb) {
      var outFile = crypto.randomBytes(20).toString('hex');

      shellExec('pdf2htmlEX --embed-javascript 0 --dest-dir /tmp ' + path + ' ' + outFile, function(err) {
        cb(err, '/tmp/' + outFile);
      });
    },
    function(outPath, cb) {
      fs.readFile(outPath, function(err, data) {
        cb(err, outPath, data);
      });
    },
    function(outPath, data, cb) {
      if(!document.metadatas) {
        document.metadatas = {};
      }

      document.metadatas.html = data.toString();
      document.binary_document_type = "pdf-html";
      cb(null, outPath);
    },
    function(outPath, cb) {
      fs.unlink(outPath, cb);
    },
  ], function(err) {
    cb(err, document);
  });
};
{'fraction_non_alphanumeric': 0.1529796511627907, 'fraction_numerical': 0.005087209302325582, 'mean_word_length': 2.9668587896253604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7454726', 'n_tokens_mistral': 892, 'n_tokens_neox': 865, 'n_words': 219}