<commit_before>'use strict';
var fs = require('fs');

var request = require('request');
var parser = require('weather-alerts-parser');
var through = require('through');

var geojson = require('../lib/index.js');


request.get('http://alerts.weather.gov/cap/us.php?x=1')
  .pipe(parser.stream())
  .pipe(geojson.stream({'stylize': true}))
  .pipe(geojson.collect({'sort': true, 'flatten': true}))
  .pipe(through(function (obj) {this.queue(JSON.stringify(obj));}))
  .pipe(fs.createWriteStream('current-nws-alerts.json'));<commit_msg>drop geocode, parameter and polygon properties when generating current geojohnson<commit_after>'use strict';
var fs = require('fs');

var request = require('request');
var parser = require('weather-alerts-parser');
var through = require('through');

var geojson = require('../lib/index.js');


function cleanProperties() {
  var remove = [
    'geocode',
    'parameter',
    'polygon'
  ];

  return through(function write(featureCollection) {
    featureCollection.features.forEach(function(feature) {
      remove.forEach(function(remove) {
        delete feature.properties[remove];
      });
    });

    this.queue(featureCollection);
  });
}

request.get('http://alerts.weather.gov/cap/us.php?x=1')
  .pipe(parser.stream())
  .pipe(geojson.stream({'stylize': true}))
  .pipe(cleanProperties())
  .pipe(geojson.collect({'sort': true, 'flatten': true}))
  .pipe(through(function (obj) {this.queue(JSON.stringify(obj));}))
  .pipe(fs.createWriteStream('current-nws-alerts.json'));
{'fraction_non_alphanumeric': 0.1911667765326302, 'fraction_numerical': 0.0013183915622940012, 'mean_word_length': 5.023809523809524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2333765', 'n_tokens_mistral': 495, 'n_tokens_neox': 483, 'n_words': 82}