<commit_before>var daff = require('daff')
var formatData = require('format-data')
var batcher = require('byte-stream')
var concat = require('concat-stream')
var through = require('through2')

var NdjsonTable = require('./ndjson_table_view.js')

var flags = new daff.CompareFlags();
//flags.allow_nested_cells = true

module.exports.fromDiff = function (changes, opts, cb) {
  if (!opts) opts = {}
  var table1 = new NdjsonTable(changes, function(row) {
    if (row && row[0]) return row[0]["value"]
    else return
  });

  var table2 = new NdjsonTable(changes, function(row) {
    if (row && row[1]) return row[1]["value"]
    else return
  });
  var output = tablesToOutput(table1, table2, opts)
  cb(null, output)
}

module.exports.fromReadStreams = function (stream1, stream2, opts, cb) {
  if (!opts) opts = {}

  function streamToTable (stream, cb) {
    var batchedStream = batcher(opts.limit)
    stream
    .pipe(through.obj(function (data, enc, next) {
      next(null, data.value)
    }))
    .pipe(formatData('ndjson'))<commit_msg>attempt to prune the changes of nulls<commit_after>var daff = require('daff')
var formatData = require('format-data')
var batcher = require('byte-stream')
var concat = require('concat-stream')
var through = require('through2')

var NdjsonTable = require('./ndjson_table_view.js')

var flags = new daff.CompareFlags();
//flags.allow_nested_cells = true

module.exports.fromDiff = function (changes, opts, cb) {
  if (!opts) opts = {}

  var data1 = []
  var data2 = []
  for (var i = 0; i < changes.length; i++) {
    var change = changes[i]
    if (change[0] && change[0]['value']) {
      data1.push(change[0]["value"])
    }
    else {
      data2.push(change[1]["value"])
    }
  }
  var table1 = new NdjsonTable(data1)
  var table2 = new NdjsonTable(data2)
  //  return row[1]["value"]

  var output = tablesToOutput(table1, table2, opts)
  cb(null, output)
}

module.exports.fromReadStreams = function (stream1, stream2, opts, cb) {
  if (!opts) opts = {}

  function streamToTable (stream, cb) {
    var batchedStream = batcher(opts.limit)
    stream
    .pipe(through.obj(function (data, enc, next) {
      next(null, data.value)
    }))
    .pipe(formatData('ndjson'))
{'fraction_non_alphanumeric': 0.15502478593961244, 'fraction_numerical': 0.013519603424966201, 'mean_word_length': 3.9006622516556293, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7271019', 'n_tokens_mistral': 766, 'n_tokens_neox': 754, 'n_words': 185}