<commit_before>#!/usr/bin/env node
var inherits = require('util').inherits;
var Writable = require('stream').Writable;
var TransformOMatic = require('./lib/transform');

function stdout() {
  if (!(this instanceof stdout)) return new stdout();
  Writable.call(this, {objectMode: true});
}

inherits(stdout, Writable);

stdout.prototype._write = function(chunk, encoding, done) {
  process.stdout.write(chunk);
  done();
};

stdout.prototype.send = function(statuscode, msg) {
  process.stderr.write(msg + '\n');
};

var usage = '\nPerform streaming data transformations on remote csv files.\n';
usage += 'More details: http://datapipes.okfnlabs.org\n\n';
usage += 'Usage: $0 [-s] [-u URL] -- [PIPELINE]';

var argv = require('optimist')
  .options({
    u: {alias: 'url', string: true, demand: true, describe: 'URL of input data.'},
    s: {alias: 'share', boolean: true, describe: 'Generate a URL to share this.'},
  })
  .usage(usage)
  .argv
;

var transformStr = argv._.join(' ');

if (argv.s) {
  var transformUrl = 'http://datapipes.okfnlabs.org/';
  transformUrl += encodeURIComponent(transformStr);
  transformUrl += '?url=';
  transformUrl += argv.url;
  var stars = Array(transformUrl.length+1).join('*');

  console.log('URL to share:');
  console.log(stars);
  console.log(transformUrl);
  console.log(stars);

  return;
}

transformStr = TransformOMatic.rejig(transformStr);

var transformers = TransformOMatic.pipeline(transformStr);

TransformOMatic.transform(stdout(), transformers, argv.url);
<commit_msg>Require at least one input argument to cli.js<commit_after>#!/usr/bin/env node
var inherits = require('util').inherits;
var Writable = require('stream').Writable;
var TransformOMatic = require('./lib/transform');

function stdout() {
  if (!(this instanceof stdout)) return new stdout();
  Writable.call(this, {objectMode: true});
}

inherits(stdout, Writable);

stdout.prototype._write = function(chunk, encoding, done) {
  process.stdout.write(chunk);
  done();
};

stdout.prototype.send = function(statuscode, msg) {
  process.stderr.write(msg + '\n');
};

var usage = '\nPerform streaming data transformations on remote csv files.\n';
usage += 'More details: http://datapipes.okfnlabs.org\n\n';
usage += 'Usage: $0 [-s] [-u URL] -- [PIPELINE]';

var argv = require('optimist')
  .options({
    u: {alias: 'url', string: true, demand: true, describe: 'URL of input data.'},
    s: {alias: 'share', boolean: true, describe: 'Generate a URL to share this.'},
  })
  .usage(usage)
  .demand(1)
  .argv
;

var transformStr = argv._.join(' ');

if (argv.s) {
  var transformUrl = 'http://datapipes.okfnlabs.org/';
  transformUrl += encodeURIComponent(transformStr);
  transformUrl += '?url=';
  transformUrl += argv.url;
  var stars = Array(transformUrl.length+1).join('*');

  console.log('URL to share:');
  console.log(stars);
  console.log(transformUrl);
  console.log(stars);

  return;
}

transformStr = TransformOMatic.rejig(transformStr);

var transformers = TransformOMatic.pipeline(transformStr);

TransformOMatic.transform(stdout(), transformers, argv.url);

{'fraction_non_alphanumeric': 0.17157656199417287, 'fraction_numerical': 0.0016186468112657818, 'mean_word_length': 4.732838589981447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1123487', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1015, 'n_words': 246}