<commit_before>#!/usr/bin/env node

'use strict';

const program = require('commander'),
  Sisyphe = require('./src/sisyphe'),
  bluebird = require('bluebird'),
  fs = bluebird.promisifyAll(require('fs'));

program
  .version('0.0.1')
  .usage('[options] <path>')
  .option('-c, --corpusname <name>', 'Choose a corpus\'s name', 'default')
  .parse(process.argv);

if (!process.argv.slice(2).length) {
  program.outputHelp();
}

const pathInput = program.args[0];
fs.statAsync(pathInput).catch((error) => {
  console.log(error);
  process.exit(1);
}).then(() => {
  return new Sisyphe({
    module: "walker-fs",
    options: {
      path: pathInput,
      corpusname: program.corpusname
    }
  },[{
    name: "Sisyphe FileType",
    module: "sisyphe-filetype"
  }, {
    name: "Sisyphe XML",
    module: "sisyphe-xml"
  }, {
    name: "Sisyphe PDF",
    module: "sisyphe-pdf"
  }, {
    name: "Sisyphe xpath",
    module: "sisyphe-xpath"
  },{
    name: "Sisyphe Output",
    module: "sisyphe-out"
  }]);
}).then((sisyphe) => {
  sisyphe.start();
});


<commit_msg>Start an update branch for sisyphe<commit_after>#!/usr/bin/env node

'use strict';

const program = require('commander'),
  Sisyphe = require('./src/sisyphe'),
  bluebird = require('bluebird'),
  fs = bluebird.promisifyAll(require('fs'));

program
  .version('0.0.1')
  .usage('[options] <path>')
  .option('-c, --corpusname <name>', 'Choose a corpus\'s name', 'default')
  .parse(process.argv);

if (!process.argv.slice(2).length) {
  program.outputHelp();
}

const pathInput = program.args[0];

var starter = {
  module: "walker-fs",
  options: {
    path: pathInput,
    corpusname: program.corpusname
  }
}
var workers = [{
  name: "Sisyphe FileType",
  module: "sisyphe-filetype"
}, {
  name: "Sisyphe XML",
  module: "sisyphe-xml"
}, {
  name: "Sisyphe PDF",
  module: "sisyphe-pdf"
}, {
  name: "Sisyphe xpath",
  module: "sisyphe-xpath"
},{
  name: "Sisyphe Output",
  module: "sisyphe-out"
}];

// This is an Update
if(!pathInput){
  starter ={
    module: "walker-elastic",
    options: {
      index: 'analyse-' + program.corpusname,
      corpusname: program.corpusname
    }
  };
  workers = [{name: "Sisyphe XML",module: "sisyphe-xml"},{name: "Sisyphe Output",module: "sisyphe-out"}]
  let sisyphe = new Sisyphe(starter,workers);
  sisyphe.start();
  return;
}

fs.statAsync(pathInput).catch((error) => {
  console.log(error);
  process.exit(1);
}).then(() => {
  return new Sisyphe(starter,workers);
}).then((sisyphe) => {
  sisyphe.start();
});



{'fraction_non_alphanumeric': 0.1973892405063291, 'fraction_numerical': 0.004746835443037975, 'mean_word_length': 3.674676524953789, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541041', 'n_tokens_mistral': 999, 'n_tokens_neox': 975, 'n_words': 183}