<commit_before>/**
 * Module dependencies.
 */

var express = require('express'),
  fs = require('fs'), 
  lazy = require('lazy'),
  spawn = require('child_process').spawn,
  app = module.exports = express.createServer(),
  // one day in milliseconds...used within a few helper functions
  od = 1000*60*60*24; 

// express configuration
app.configure(function(){
  app.set('views', __dirname + '/views');
  app.set('view engine', 'jade');
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(app.router);
  app.use(express.static(__dirname + '/public'));
});

app.configure('development', function(){
  app.use(express.errorHandler({ dumpExceptions: true, showStack: true })); 
});

app.configure('production', function(){
  app.use(express.errorHandler()); 
});

// helper functions
Array.max = function( array ){
    return Math.max.apply( Math, array );
};

Array.min = function( array ){
    return Math.min.apply( Math, array );
};

var checkRange = function(x, n, m) {
    if (x >= n && x <= m) { return x; }
    else { return !x; }
};

var isConfig = function(x) {
  var re = /.*\.json/;
  return String(x).match(re);
};

var dateToStr = function (d) {
  var y = d.getFullYear().toString(),
    m = '0' + (d.getMonth() +1).toString(),
    s = '0' + d.getDate().toString();
    return y + '-' + m.substring(m.length - 2) + '-' + s.substring(s.length - 2);
};

var daysAfter = function(d,n) {
  var b = new Date(d);
  var e = new Date(b.getTime() + (n*od));
  return dateToStr(e);
};

/* 
var startOfWeek = function(d) {
  var t = d || new Date();  
  var td = new Date(t.getFullYear(), t.getMonth(), t.getDate());
  return new Date(td.getTime() - (td.getDay() * od));
};

var startOfPreviousWeek = function(d) {
  return new Date(startOfWeek(d).getTime() - (7 * od));
};

var endOfWeek = function(d) {
  return new Date(startOfWeek(d).getTime() + (7 * od));
};

var endOfPreviousWeek = function(d) {
  return new Date(startOfPreviousWeek(d).getTime() + (7 * od));
};
*/

function getSeriesConfig(vname, selectedOptions, next) {
  var results = [];

  fs.readFile("./tsv/" + vname + ".json", function(err,buffer) {
    var c = JSON.parse(buffer),
      // create this array based on those that have "default" eh?
      b = ['fromDate','series','selectionScope'];

    for (var i in b) {
      var k = b[i],
        re = new RegExp("~"+k, "g");

      // set the value of config element c[k]
      // based on options selected in the ui
      // otherwise default settings
      if (selectedOptions && selectedOptions[k]) {
        var s = selectedOptions[k];

        if (s) { (s instanceof Array) ? c[k].value = s.join(',') : c[k].value = s; }
      } else {
        if (c[k]['default']['value']) {
          c[k].value = c[k]['default']['value'];
        } else if (c[k]['default']['function']) {
          c[k].value = eval(c[k]['default']['function']);
        }
      }

      // replacements based on the value of c[k]
      c = JSON.parse(JSON.stringify(c).replace(re, "'"+c[k].value+"'"));
    }

    results.push(c);
    next(results);
  });

}

function getSeriesData (vname, selectedOptions, next) {
  getSeriesConfig(vname, selectedOptions, function(data) {
    var c = data.shift(),
      // todo: get rid of this ugliness calling a shell script here...
      // if we tuck it away in the config for now at least that gives us options 
      // for calling different stuff...still ugly I know
      csv = spawn('./tsv/mssql.sh', [c.cmd]);

    console.log("baz");
    console.log(c.cmd);

    new lazy(csv.stdout).lines.map(String).map(function (line){
      return line.split(',');
    }).join(function (results) {
      results.unshift(c);
      next(results);
    });
  });
};

function buildSeries (vname, selectedOptions, next) {
  getSeriesData(vname, selectedOptions, function(data) {
    var q = data.shift(), // q for qonfig...hell, running out of letters for my alphabet soup
      header = data[0],
      t = [], // temporary array will have named indexes 
      results = []; // will be sent to next, must be JSON stringifiable

    // todo: need to drive this by the data rather than the config
    // q['xLabs'] = getXlabs(parseDateString(q.fromDate.value), parseDateString(q.untilDate.value));

    for(var i = 1; i < data.length ; i++) {
      var c = data[i];
      var k = c[0];
      var r = "";
      // using an array with named index for convenience 
      // in order to collect all data points for a given key 
      t[k] = t[k] || {}; 
      t[k][header[0]] = c[0];<commit_msg>trying to figure out why node eventually stops delivering data to the client<commit_after>/**
 * Module dependencies.
 */

var express = require('express'),
  fs = require('fs'), 
  lazy = require('lazy'),
  spawn = require('child_process').spawn,
  app = module.exports = express.createServer(),
  // one day in milliseconds...used within a few helper functions
  od = 1000*60*60*24; 

// express configuration
app.configure(function(){
  app.set('views', __dirname + '/views');
  app.set('view engine', 'jade');
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(app.router);
  app.use(express.static(__dirname + '/public'));
});

app.configure('development', function(){
  app.use(express.errorHandler({ dumpExceptions: true, showStack: true })); 
});

app.configure('production', function(){
  app.use(express.errorHandler()); 
});

// helper functions
Array.max = function( array ){
    return Math.max.apply( Math, array );
};

Array.min = function( array ){
    return Math.min.apply( Math, array );
};

var checkRange = function(x, n, m) {
    if (x >= n && x <= m) { return x; }
    else { return !x; }
};

var isConfig = function(x) {
  var re = /.*\.json/;
  return String(x).match(re);
};

var dateToStr = function (d) {
  var y = d.getFullYear().toString(),
    m = '0' + (d.getMonth() +1).toString(),
    s = '0' + d.getDate().toString();
    return y + '-' + m.substring(m.length - 2) + '-' + s.substring(s.length - 2);
};

var daysAfter = function(d,n) {
  var b = new Date(d);
  var e = new Date(b.getTime() + (n*od));
  return dateToStr(e);
};

/* 
var startOfWeek = function(d) {
  var t = d || new Date();  
  var td = new Date(t.getFullYear(), t.getMonth(), t.getDate());
  return new Date(td.getTime() - (td.getDay() * od));
};

var startOfPreviousWeek = function(d) {
  return new Date(startOfWeek(d).getTime() - (7 * od));
};

var endOfWeek = function(d) {
  return new Date(startOfWeek(d).getTime() + (7 * od));
};

var endOfPreviousWeek = function(d) {
  return new Date(startOfPreviousWeek(d).getTime() + (7 * od));
};
*/

function getSeriesConfig(vname, selectedOptions, next) {
  var results = [];

  fs.readFile("./tsv/" + vname + ".json", function(err,buffer) {
    var c = JSON.parse(buffer),
      // create this array based on those that have "default" eh?
      b = ['fromDate','series','selectionScope'];

    for (var i in b) {
      var k = b[i],
        re = new RegExp("~"+k, "g");

      // set the value of config element c[k]
      // based on options selected in the ui
      // otherwise default settings
      if (selectedOptions && selectedOptions[k]) {
        var s = selectedOptions[k];

        if (s) { (s instanceof Array) ? c[k].value = s.join(',') : c[k].value = s; }
      } else {
        if (c[k]['default']['value']) {
          c[k].value = c[k]['default']['value'];
        } else if (c[k]['default']['function']) {
          c[k].value = eval(c[k]['default']['function']);
        }
      }

      // replacements based on the value of c[k]
      c = JSON.parse(JSON.stringify(c).replace(re, "'"+c[k].value+"'"));
    }

    results.push(c);
    next(results);
  });

}

function getSeriesData (vname, selectedOptions, next) {
  getSeriesConfig(vname, selectedOptions, function(data) {
    var c = data.shift(),
      // todo: get rid of this ugliness calling a shell script here...
      // if we tuck it away in the config for now at least that gives us options 
      // for calling different stuff...still ugly I know
      csv = spawn('./tsv/mssql.sh', [c.cmd]);

    console.log("baz");
    console.log(c.cmd);

    csv.stdout.on('data', function (data) {
      console.log('stdout: ' + data);
      console.log('buz');
    });

    new lazy(csv.stdout).lines.map(String).map(function (line){
      return line.split(',');
    }).join(function (results) {
      results.unshift(c);
      next(results);
    });
  });
};

function buildSeries (vname, selectedOptions, next) {
  getSeriesData(vname, selectedOptions, function(data) {
    var q = data.shift(), // q for qonfig...hell, running out of letters for my alphabet soup
      header = data[0],
      t = [], // temporary array will have named indexes 
      results = []; // will be sent to next, must be JSON stringifiable

    // todo: need to drive this by the data rather than the config
    // q['xLabs'] = getXlabs(parseDateString(q.fromDate.value), parseDateString(q.untilDate.value));

    for(var i = 1; i < data.length ; i++) {
      var c = data[i];
      var k = c[0];
      var r = "";
      // using an array with named index for convenience 
      // in order to collect all data points for a given key 
      t[k] = t[k] || {}; 
      t[k][header[0]] = c[0];
{'fraction_non_alphanumeric': 0.17942635324563294, 'fraction_numerical': 0.004960103515203796, 'mean_word_length': 2.943452380952381, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1226555', 'n_tokens_mistral': 3097, 'n_tokens_neox': 3022, 'n_words': 835}