<commit_before>var fs = require('fs');

var sys = require('sys');
var express = require('express');
var csv = require('csv');
var Stream = require('stream');
var request = require('request');
var path = require('path');
var nunjucks = require('nunjucks')
var marked = require('marked');
var _ = require('underscore');

var app = express();

//CORS middleware
var CORSSupport = function(req, res, next) {
  res.header('Access-Control-Allow-Origin', '*');
  res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE');
  res.header('Access-Control-Allow-Headers', 'Content-Type');
  // intercept OPTIONS method
  if ('OPTIONS' == req.method) {
    res.send(200);
  }
  else {
    next();
  }
}

app.configure(function(){
  app.set('port', process.env.PORT || 5000);
  app.set('views', __dirname + '/templates');
  app.use(express.logger('dev'));
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(CORSSupport);
  app.use(express.static(path.join(__dirname, 'public')));
});

var env = new nunjucks.Environment(new nunjucks.FileSystemLoader('templates'));
env.express(app);

function noop(arg){return arg}

function convert(instream, outstream, mapfunc) {
  var outcsv = csv();

  var initial = mapfunc(null, 'pre');
  if (initial){
    outstream.write(initial);
  }

  outcsv
    .from.stream(instream)
    .to.stream(outstream, {end: false})
    .transform(mapfunc)
    .on('end', function(count) {
      var out = mapfunc(null, 'end');
      if (out) {
        outstream.write(out);
      }
      outstream.end();
    })
  ;
}

// Collection of staticmethod(ish) things that will
// return a
var Transformations = {

  // HOF to return our HEAD transformation
  head: function(call){
    var lines = 0;
    var number = call[2];
    if(!number){
      var number = 40;
    }
    return function(row, idx){
      if(lines < number){
        lines += 1;
        return row
      }else{
        return null;
      }
    }
  },

  // HOF to return a transformation that will cut columns
  // Accepts a comma separated list of column positions to remove
  // This is 0-indexed.
  cut: function(call){
    var columns = call[1] || "";
    var parts = columns.split(',');

    // Convert the indices to ints
    var idxes = _.chain(parts)
                 .map(function(part){
                   if (part.indexOf('-') == -1) {
                     return [parseInt(part)];
                   } else {
                     range = part.split('-');
                     return _.range(parseInt(range[0]), parseInt(range[1])+1);
                   }
                 })
                 .flatten()
                 .uniq()
                 .value();

    return function(row, index){
      if(index ==  'pre'){
        return null;
      }else if(index == 'post'){
        return null;
      }
      if (row === null) {
        return null;
      }

      // delete the values at the specific position
      _.each(idxes, function(position) {
        delete row[position]
      })

      return _.without(row, undefined)
    }
  },

  // HOF to return a transformation that will grep for a pattern
  grep: function(call){
    var pattern = call[1];
    return function(row, index){
      if(index == 0){
        return row
      }
      var match = false;
      _.each(row, function(value){
        if(value.indexOf(pattern) != -1){
          match = true;<commit_msg>Refactor a bit (as @rossjones suggests in #39)<commit_after>var fs = require('fs');

var sys = require('sys');
var express = require('express');
var csv = require('csv');
var Stream = require('stream');
var request = require('request');
var path = require('path');
var nunjucks = require('nunjucks')
var marked = require('marked');
var _ = require('underscore');

var app = express();

//CORS middleware
var CORSSupport = function(req, res, next) {
  res.header('Access-Control-Allow-Origin', '*');
  res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE');
  res.header('Access-Control-Allow-Headers', 'Content-Type');
  // intercept OPTIONS method
  if ('OPTIONS' == req.method) {
    res.send(200);
  }
  else {
    next();
  }
}

app.configure(function(){
  app.set('port', process.env.PORT || 5000);
  app.set('views', __dirname + '/templates');
  app.use(express.logger('dev'));
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(CORSSupport);
  app.use(express.static(path.join(__dirname, 'public')));
});

var env = new nunjucks.Environment(new nunjucks.FileSystemLoader('templates'));
env.express(app);

function noop(arg){return arg}

function toIntArr(str){
  var parts = str.split(',');
  return _.chain(parts)
          .map(function(part){
            if (part.indexOf('-') == -1) {
              return [parseInt(part)];
            } else {
              range = part.split('-');
              return _.range(parseInt(range[0]), parseInt(range[1])+1);
            }
          })
          .flatten()
          .uniq()
          .value();
}

function convert(instream, outstream, mapfunc) {
  var outcsv = csv();

  var initial = mapfunc(null, 'pre');
  if (initial){
    outstream.write(initial);
  }

  outcsv
    .from.stream(instream)
    .to.stream(outstream, {end: false})
    .transform(mapfunc)
    .on('end', function(count) {
      var out = mapfunc(null, 'end');
      if (out) {
        outstream.write(out);
      }
      outstream.end();
    })
  ;
}

// Collection of staticmethod(ish) things that will
// return a
var Transformations = {

  // HOF to return our HEAD transformation
  head: function(call){
    var lines = 0;
    var number = call[2];
    if(!number){
      var number = 40;
    }
    return function(row, idx){
      if(lines < number){
        lines += 1;
        return row
      }else{
        return null;
      }
    }
  },

  // HOF to return a transformation that will cut columns
  // Accepts a comma separated list of column positions to remove
  // This is 0-indexed.
  cut: function(call){
    var columns = call[1] || "";

    // Convert the indices to ints
    var idxes = toIntArr(columns);

    return function(row, index){
      if(index ==  'pre'){
        return null;
      }else if(index == 'post'){
        return null;
      }
      if (row === null) {
        return null;
      }

      // delete the values at the specific position
      _.each(idxes, function(position) {
        delete row[position]
      })

      return _.without(row, undefined)
    }
  },

  // HOF to return a transformation that will grep for a pattern
  grep: function(call){
    var pattern = call[1];
    return function(row, index){
      if(index == 0){
        return row
      }
      var match = false;
      _.each(row, function(value){
        if(value.indexOf(pattern) != -1){
          match = true;
{'fraction_non_alphanumeric': 0.154763657782359, 'fraction_numerical': 0.0064791635988808716, 'mean_word_length': 2.442473390775469, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1278697', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2088, 'n_words': 532}