<commit_before>function processFile(file) {
  var reader = new FileReaderSync();
  var experiment = JSON.parse(reader.readAsText(file));
  var columns = Object.keys(experiment.data[0]);
  var csv = [columns.join(",") + "\n"];
  csv = csv.concat(experiment.data.map(function(row) {
    return columns.map(function(field) {
      var v = row[field];
      if (v === undefined) {
        console.log("field " + field + " missing");
        return "";
      } else
        return v.toString();
    }).join(',') + "\n";
  }));
  return new Blob(csv, {type: "text/csv"});
}

onmessage = function (e) {
  var data = e.data;

  switch(data.action) {
    case "processFile":
      var csv = processFile(data.file);
      postMessage({csv: csv});
      break;
  };
};<commit_msg>handle multiple records in a single file correctly<commit_after>function processFile(file) {
  var reader = new FileReaderSync();
  var text = reader.readAsText(file);
  text.trim().split("\n").forEach(function (experiment) { processJSON(JSON.parse(experiment)) });
}

function processJSON(experiment) {
  var columns = Object.keys(experiment.data[0]);
  var csv = [columns.join(",") + "\n"];
  csv = csv.concat(experiment.data.map(function(row) {
    return columns.map(function(field) {
      var v = row[field];
      if (v === undefined) {
        console.log("field " + field + " missing");
        return "";
      } else
        return v.toString();
    }).join(',') + "\n";
  }));
  blob = new Blob(csv, {type: "text/csv"});
  postMessage({csv: blob});
}

onmessage = function (e) {
  var data = e.data;

  switch(data.action) {
    case "processFile":
      processFile(data.file);
      break;
  };
};
{'fraction_non_alphanumeric': 0.18035714285714285, 'fraction_numerical': 0.0011904761904761906, 'mean_word_length': 3.1403940886699506, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5276373', 'n_tokens_mistral': 555, 'n_tokens_neox': 546, 'n_words': 120}