<commit_before>#! /usr/bin/env node

var h           = require('highland')
  , _           = require('lodash')
  , program     = require('commander')
  , XLSX        = require('xlsx')
  , request     = require('superagent')
  , through2    = require('through2')
  , mappedColumns;

var exceltoJSONStream = function(file){
  var workbook = XLSX.readFile(file);
  var ws = workbook.Sheets[workbook.SheetNames[0]];
  var json = XLSX.utils.sheet_to_json(ws);
  return h(json);
};

var lowerCasedKeys = function(row){
  const result = {};
  _.keys(row).forEach(key => result[key.toLowerCase()] = row[key]);
  return result;
};

var mapColumns = function(row){
  const result = _.assign({}, row);
  const inverted = _.invert(mappedColumns);

  _.values(mappedColumns).forEach( val => {
    result[inverted[val]] = row[val];
    delete result[val]
  });

  return result;
};

function parseMapCols(val){
  var result = {};
  var val = val.replace(/\s|\"|\{|\}|\[|\]/g, '');
  val.split(',').forEach(function(e){
    var parts = e.split(':');
    result[parts[0]] = parts[1] === 'null' ? null : parts[1];
  });
  return result;
}

program
.usage('[options] <file>')
.arguments('<file>')
.option('-u, --url <http request target>', 'URL to work with.')
.option('-c, --mapcols [override1:original1, ...]', 'Optional column headers to override originals in xls/csv.', parseMapCols)
.action(function(file) {
  mappedColumns = program.mapcols;

  exceltoJSONStream(file)
  .map(lowerCasedKeys)
  .map(mapColumns)
  .map(JSON.stringify)
  .pipe(through2.obj(function(payload, h, next){
    request
    .post(program.url)
    .send(payload)
    .set('Accept', 'application/json')
    .set('Content-Type', 'application/json')
    .end(function(err, res){
      next(err, JSON.stringify(res));
    });
  }))
  .pipe(process.stdout);
})
.parse(process.argv);<commit_msg>Index -> is now the library API.<commit_after>'use strict';

const h           = require('highland')
    , _           = require('lodash')
    , XLSX        = require('xlsx')
    , request     = require('superagent')
    , through2    = require('through2');

const workbookToStream = (workbook, opts) => {
  const ws = workbook.Sheets[workbook.SheetNames[0]];
  const json = XLSX.utils.sheet_to_json(ws);
  return h(json);
};

const lowerCasedKeys = (row) => {
  const result = {};
  _.keys(row).forEach(key => result[key.toLowerCase()] = row[key]);
  return result;
};

const override = (overriden, row) => {
  const result = _.assign({}, row);
  const inverted = _.invert(overriden);

  _.values(overriden).forEach( val => {
    result[inverted[val]] = row[val];
    delete result[val]
  });

  return result;
};

const ignore = (ignored, row) => {
  const result = _.assign({}, row);
  ignored.forEach( i => { delete result[i] });
  return result;
};

function Slingg(opts) {
  if (!(this instanceof Slingg)) return new Slingg(opts);

  opts || (opts = {});
  if(!opts.url){ throw new Error('Destination url is missing!'); }

  this.url = opts.url;
  this.headers = opts.headers;
}

Slingg.prototype.hasOverrideHeaders = function () {
  return this.headers && this.headers.override;
};

Slingg.prototype.hasIgnoreHeaders = function () {
  return this.headers && this.headers.ignore;
};

Slingg.prototype.extractWorkbookFromPath = function (filePath) {
  this.workbook = XLSX.readFile(filePath);
  return this;
};

Slingg.prototype.extractWorkbookFromBuffer = function (buffer) {
  this.workbook = XLSX.read(buffer);
  return this;
};

Slingg.prototype.createBaseStream = function () {
  const ws = this.workbook.Sheets[this.workbook.SheetNames[0]];
  this.baseStream = h(XLSX.utils.sheet_to_json(ws));
  return this;
};

Slingg.prototype.start = function () {
  return this.baseStream
    .map(lowerCasedKeys)
    .map((row) => { return this.hasOverrideHeaders() ? override(this.headers.override, row) : row; })
    .map((row) => { return this.hasIgnoreHeaders() ? ignore(this.headers.ignore, row) : row; })
    .doto(console.log)
    .map(JSON.stringify)
    .pipe(through2.obj((payload, _, next) => {
      request
      .post(this.url)
      .send(payload)
      .set('Accept', 'application/json')
      .set('Content-Type', 'application/json')
      .end(function(err, res){
        next(err, JSON.stringify(res));
      });
    }));
};

Slingg.fromPath = (filePath, opts) => {
  const obj = new Slingg(opts);
  return obj
  .extractWorkbookFromPath(filePath)
  .createBaseStream();
};

Slingg.fromBuffer = (buffer, opts) => {
  const obj = new Slingg(opts);
  return obj
  .extractWorkbookFromBuffer(buffer)
  .createBaseStream();
};

module.exports = Slingg;
{'fraction_non_alphanumeric': 0.17540060632308357, 'fraction_numerical': 0.0030316154179298397, 'mean_word_length': 3.642211055276382, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992449', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1578, 'n_words': 306}