<commit_before>const csv = require('csv');
const JSONStream = require('JSONStream');

function toObject(row) {
  function parseTime(s) {
    const parts = s.split(':');
    const now = new Date();
    now.setHours(parts[0], parts[1], 0, 0);
    return now;
  }

  function parseDuration(s) {
    const parts = s.trim().split('-');
    if (parts.length < 2) return null;
    return {'start': parseTime(parts[0]), 'stop': parseTime(parts[1])};
  }
  var obj = parseDuration(row[0]);
  if (!obj) return null;
  obj.space = {'title': row[1], 'speaker': row[2]};
  obj.tab =   {'title': row[3], 'speaker': row[4]};
  return obj;
}

// Parses cvs stream and converts to json usable by frontend
function cvs2json(input, output) {
  input
    .pipe(csv.parse())
    .pipe(csv.transform(toObject))
    .pipe(JSONStream.stringify())
    .pipe(output);
}

module.exports = { cvs2json };
<commit_msg>refactor: moving as inner funcs<commit_after>const csv = require('csv');
const JSONStream = require('JSONStream');

// Parses cvs stream and converts to json usable by frontend
function cvs2json(input, output) {
  function toObject(row) {
    function parseTime(s) {
      const parts = s.split(':');
      const now = new Date();
      now.setHours(parts[0], parts[1], 0, 0);
      return now;
    }

    function parseDuration(s) {
      const parts = s.trim().split('-');
      if (parts.length < 2) return null;
      return {'start': parseTime(parts[0]), 'stop': parseTime(parts[1])};
    }
    var obj = parseDuration(row[0]);
    if (!obj) return null;
    obj.space = {'title': row[1], 'speaker': row[2]};
    obj.tab =   {'title': row[3], 'speaker': row[4]};
    return obj;
  }

  input
    .pipe(csv.parse())
    .pipe(csv.transform(toObject))
    .pipe(JSONStream.stringify())
    .pipe(output);
}

module.exports = { cvs2json };

{'fraction_non_alphanumeric': 0.17759562841530055, 'fraction_numerical': 0.015300546448087432, 'mean_word_length': 3.433414043583535, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7610061', 'n_tokens_mistral': 642, 'n_tokens_neox': 628, 'n_words': 158}