<commit_before>var through = require('through2');

module.exports = function(){
  var stream = through(function(chunk, enc, cb){

    try{
      var obj = JSON.parse(chunk.toString());
      var coords = obj.geometry.coordinates;
      var props = obj.properties;
      var x = coords[0];
      var y = coords[1];
      var addr = props.address;
      var alt = props.alt_address;
    }catch(e){
      return process.nextTick(function(){
        return cb(new Error('Couldn\'t backup invalid GeoJSON feature: ' + chunk.toString()));
      });
    }

    return cb(null, [x, y, addr, alt].join(',') + '\n');
  });

  //write headers
  stream.write(JSON.stringify({
    type: "Feature",
    geometry: {
      type: "Point",
      coordinates: ["x", "y"]
    },
    properties: {
      "address": "address",
      "alt_address": "alt_address"
    }
  }));

  return stream;
};<commit_msg>update jsonToCsv to new schema<commit_after>var through = require('through2');

module.exports = function(){
  var stream = through(function(chunk, enc, cb){

    try{
      var obj = JSON.parse(chunk.toString());
      var coords = obj.geometry.coordinates;
      var props = obj.properties;
      var x = coords[0];
      var y = coords[1];
    }catch(e){
      return process.nextTick(function(){
        return cb(new Error('Couldn\'t backup invalid GeoJSON feature: ' + chunk.toString()));
      });
    }

    return cb(null, [x, y, props.address, props.number, props.street, props.city, props.state, props.zip].join(',') + '\n');
  });

  //write headers
  stream.write(JSON.stringify({
    type: "Feature",
    geometry: {
      type: "Point",
      coordinates: ["x", "y"]
    },
    properties: {
      "address": "address",
      "number": "number",
      "street": "street",
      "city": "city",
      "state": "state",
      "zip": "zip"
    }
  }));

  return stream;
};
{'fraction_non_alphanumeric': 0.17326203208556148, 'fraction_numerical': 0.0032085561497326204, 'mean_word_length': 2.633009708737864, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5064005', 'n_tokens_mistral': 603, 'n_tokens_neox': 593, 'n_words': 146}