<commit_before>var turf = require('turf')
var fs = require('fs')

// wow. such vars.
var tractPolys = JSON.parse(fs.readFileSync(process.argv[2]));
var piranhas = JSON.parse(fs.readFileSync(process.argv[3]));
var piranhaPolys = {"type":"FeatureCollection","features":[]};
var tractsEaten = {"type":"FeatureCollection","features":[]};

console.log('Processing ' + process.argv[3] + ' with ' + piranhas.features.length + ' water features');

if (piranhas.features) {
  // Isolate the water polygons from all that other osm noise
  for (var p = 0; p < piranhas.features.length; p++) {
    if (piranhas.features[p].geometry.type == 'Polygon' || piranhas.features[p].geometry.type == 'MultiPolygon') {
      piranhaPolys.features.push(piranhas.features[p]);
    } 
  }

  // loop through all tracts looking for intersection
  for (var i = 0; i < tractPolys.features.length; i++) {
    // chomp water from tract polygon if intersecting
    //console.log('Running '+ tractPolys.features[i].properties.TRACTCE)
    if (turf.intersect(piranhaPolys.features[0],tractPolys.features[i])) {
      console.log('Tract ' + tractPolys.features[i].properties.TRACTCE + ' is in the water. Chomping . . .')
      tractsEaten.features.push(turf.erase(tractPolys.features[i],piranhaPolys.features[0]));
    } 
    // otherwise just write the tract polygon as is
    else {
      //console.log('Tract ' + tractPolys.features[i].properties.TRACTCE + ' is dry.')
      tractsEaten.features.push(tractPolys.features[i])
    }
  }

  //cut and print to file
  fs.writeFileSync(process.argv[2], JSON.stringify(tractsEaten));
} else {
  fs.writeFileSync(process.argv[2], JSON.stringify(tractPolys));
}<commit_msg>buffer strategies<commit_after>var turf = require('turf')
var fs = require('fs')

// wow. such vars.
var tractPolys = JSON.parse(fs.readFileSync(process.argv[2]));
var piranhas = JSON.parse(fs.readFileSync(process.argv[3]));
var piranhaPolys = {"type":"FeatureCollection","features":[]};
var tractsEaten = {"type":"FeatureCollection","features":[]};

console.log('Processing ' + process.argv[3] + ' with ' + piranhas.features.length + ' water features');

if (piranhas.features.length > 0) {
  // Isolate the water polygons from all that other osm noise
  for (var p = 0; p < piranhas.features.length; p++) {
    if (piranhas.features[p].geometry.type == 'Polygon' || piranhas.features[p].geometry.type == 'MultiPolygon') {
      piranhaPolys.features.push(piranhas.features[p]);
    } 
  }

  try {
    // expand everything by 50m to catch intersections
    // piranhaPolys = turf.buffer(piranhaPolys,0.05,'kilometers');
    // dissolve the water features
    piranhaPolys = turf.merge(piranhaPolys);
  }
  catch (e) {
    console.log(e);
  }

  // loop through all tracts looking for intersection
  for (var i = 0; i < tractPolys.features.length; i++) {
    // chomp water from tract polygon if intersecting
    if (turf.intersect(piranhaPolys,tractPolys.features[i])) {
      console.log('Tract ' + tractPolys.features[i].properties.TRACTCE + ' is in the water. Chomping . . .')
      tractsEaten.features.push(turf.erase(tractPolys.features[i],piranhaPolys));
    } 
    // otherwise just write the tract polygon as is
    else {
      //console.log('Tract ' + tractPolys.features[i].properties.TRACTCE + ' is dry.')
      tractsEaten.features.push(tractPolys.features[i])
    }
  }

  //cut and print to file
  fs.writeFileSync(process.argv[2], JSON.stringify(tractsEaten));
} else {
  fs.writeFileSync(process.argv[2], JSON.stringify(tractPolys));
}
{'fraction_non_alphanumeric': 0.15597626448149193, 'fraction_numerical': 0.006216445323537723, 'mean_word_length': 3.978902953586498, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '895908', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1170, 'n_words': 237}