<commit_before>var csv = require('fast-csv');
var async = require('async');
var redis = require('redis-url');
var request = require('request');

var fs = require('fs');
var querystring = require('querystring');
var stream = require('stream');
var util = require('util');

var cacheVersion = 0;

var blackholeCacheQueue = async.queue(function(task, callback){
  setTimeout(function(){
    task.callback(null, 'null');
    callback();
  }, 100);
}, 512);
blackholeCacheQueue.set = function(key, value, callback){
  if (callback) process.nextTick(callback);
};

var redisClient;
var redisCacheQueue = async.queue(function(task, callback){
  redisClient.get(task.key, function(err, reply){
    process.nextTick(function(){
      task.callback(err, reply);
      callback();
    });
  });
}, 64);
redisCacheQueue.set = function(key, value, callback){
  redisClient.set(key, value, callback);
};

var zip5 = function(inputZip){
  var zip = inputZip;
  if (!zip) return '';

  zip = zip.split('-')[0];

  if (zip.length >= 7) {
    zip = zip.substr(0, zip.length-4);
  }

  while (zip.length < 5) {
    zip = '0'+zip;
  }

  var intZip = zip*1;

  if (zip.length != 5 || intZip < 501 || intZip > 99950) {
    throw new Error('bad zip: '+inputZip);
  }

  return zip;
};

var cacheKeyGenerator = function(row, options){
  var zipCode = row[options.zipcodeCol];
  if (zipCode) {
    try {
      zipCode = zip5(zipCode);
    } catch (err) {
      zipCode = '';
    }
  }
  var key = cacheVersion + ':' + row[options.streetCol] + ':' + (
    zipCode
      ? zipCode
      : row[options.cityCol] + ':' +
        row[options.stateCol]
  );
  key = key.toUpperCase();
  return key;
};

var structuredRow = function(structure, row, prefix, suffix){
  var output = {};
  var addField = function(header, value){
    if (prefix) {
      header = prefix+header;
    }
    if (suffix) {
      header = header+suffix;
    }
    output[header] = value || null;
  };

  for (var key in structure) {
    var value = structure[key];
    if (typeof value == 'object') {
      for (var subkey in value) {
        var subvalue = value[subkey];
        if (subvalue) {
          var header = (subvalue === true) ? subkey : subvalue;
          var rowValue = (typeof row[key] == 'object') ? row[key][subkey] : null;
          addField(header, rowValue);
        }
      }
    } else if (value) {
      var header = (value === true) ? key : value;
      addField(header, row[key]);
    }
  }
  return output;
};

var Smartystreets = function(options){
  stream.PassThrough.apply(this);

  var inputStream = csv({headers: true});
  var outputStream = csv.createWriteStream({headers: true});

  this.on('pipe', function(source){
    //redirect pipe(this) to pipe(inputStream)
    source.unpipe(this);
    source.pipe(inputStream);
  });
  this.pipe = function(destination, pipeOptions){
    //when asked to pipe this into something, pipe outputStream instead
    return outputStream.pipe(destination, pipeOptions);
  };

  var columnList;
  var addColumns = function(row){
    var out = [];
    columnList.forEach(function(column){
      out.push(row[column]);
    });
    return out;
  };

  var progress = {
    total: 0,
    cached: 0,
    geocoded: 0
  };

  var pool = {
    maxSockets: 1024
  };

  var cacheQueue = blackholeCacheQueue;

  if (options.redis) {
    redisClient = redis.createClient( (typeof options.redis == 'string') ? options.redis : undefined );
    cacheQueue = redisCacheQueue;
  }

  var self = this;

  //geocoder is a queue which takes arrays of objects, up to 99 at a time
  var geocoder = async.queue(function(rows, callback){
    //turn the rows into something smartystreets can read
    var addressList = rows.map(function(d){
      return {
        street:   d[options.streetCol] || '',
        city:     d[options.cityCol] || '',
        state:    d[options.stateCol] || '',
        zipcode:  d[options.zipcodeCol] || ''
      };
    }).filter(function(d){
      if (!d.street) {
        return false;
      }
      if (!d.zipcode || (!d.city && !d.state)) {
        return false;
      }
      return true;
    });

    if (!addressList.length) {
      process.nextTick(callback);
      return;
    }

    //send the post request
    request.post({
      uri: 'https://api.smartystreets.com/street-address?'+querystring.stringify({
        'auth-id': options.authId,
        'auth-token': options.authToken
      }),
      json: addressList,
      forever: true, //use http keepAlive
      pool: pool //don't use the default connection pool (for performance)
    }, function(err, response, body){
      if (err || response.statusCode != 200) {
        if ((response && response.statusCode === 504) || (err && (err.code == 'ECONNRESET' || err.code == 'ENOTFOUND'))) {
          console.error('connection failed, retrying chunk');
          geocoder.push([rows], function(){
            self.emit('progress', progress);
          });<commit_msg>indicate connection problem when there is one<commit_after>var csv = require('fast-csv');
var async = require('async');
var redis = require('redis-url');
var request = require('request');

var fs = require('fs');
var querystring = require('querystring');
var stream = require('stream');
var util = require('util');

var cacheVersion = 0;

var blackholeCacheQueue = async.queue(function(task, callback){
  setTimeout(function(){
    task.callback(null, 'null');
    callback();
  }, 100);
}, 512);
blackholeCacheQueue.set = function(key, value, callback){
  if (callback) process.nextTick(callback);
};

var redisClient;
var redisCacheQueue = async.queue(function(task, callback){
  redisClient.get(task.key, function(err, reply){
    process.nextTick(function(){
      task.callback(err, reply);
      callback();
    });
  });
}, 64);
redisCacheQueue.set = function(key, value, callback){
  redisClient.set(key, value, callback);
};

var zip5 = function(inputZip){
  var zip = inputZip;
  if (!zip) return '';

  zip = zip.split('-')[0];

  if (zip.length >= 7) {
    zip = zip.substr(0, zip.length-4);
  }

  while (zip.length < 5) {
    zip = '0'+zip;
  }

  var intZip = zip*1;

  if (zip.length != 5 || intZip < 501 || intZip > 99950) {
    throw new Error('bad zip: '+inputZip);
  }

  return zip;
};

var cacheKeyGenerator = function(row, options){
  var zipCode = row[options.zipcodeCol];
  if (zipCode) {
    try {
      zipCode = zip5(zipCode);
    } catch (err) {
      zipCode = '';
    }
  }
  var key = cacheVersion + ':' + row[options.streetCol] + ':' + (
    zipCode
      ? zipCode
      : row[options.cityCol] + ':' +
        row[options.stateCol]
  );
  key = key.toUpperCase();
  return key;
};

var structuredRow = function(structure, row, prefix, suffix){
  var output = {};
  var addField = function(header, value){
    if (prefix) {
      header = prefix+header;
    }
    if (suffix) {
      header = header+suffix;
    }
    output[header] = value || null;
  };

  for (var key in structure) {
    var value = structure[key];
    if (typeof value == 'object') {
      for (var subkey in value) {
        var subvalue = value[subkey];
        if (subvalue) {
          var header = (subvalue === true) ? subkey : subvalue;
          var rowValue = (typeof row[key] == 'object') ? row[key][subkey] : null;
          addField(header, rowValue);
        }
      }
    } else if (value) {
      var header = (value === true) ? key : value;
      addField(header, row[key]);
    }
  }
  return output;
};

var Smartystreets = function(options){
  stream.PassThrough.apply(this);

  var inputStream = csv({headers: true});
  var outputStream = csv.createWriteStream({headers: true});

  this.on('pipe', function(source){
    //redirect pipe(this) to pipe(inputStream)
    source.unpipe(this);
    source.pipe(inputStream);
  });
  this.pipe = function(destination, pipeOptions){
    //when asked to pipe this into something, pipe outputStream instead
    return outputStream.pipe(destination, pipeOptions);
  };

  var columnList;
  var addColumns = function(row){
    var out = [];
    columnList.forEach(function(column){
      out.push(row[column]);
    });
    return out;
  };

  var progress = {
    total: 0,
    cached: 0,
    geocoded: 0
  };

  var pool = {
    maxSockets: 1024
  };

  var cacheQueue = blackholeCacheQueue;

  if (options.redis) {
    redisClient = redis.createClient( (typeof options.redis == 'string') ? options.redis : undefined );
    cacheQueue = redisCacheQueue;
  }

  var self = this;

  //geocoder is a queue which takes arrays of objects, up to 99 at a time
  var geocoder = async.queue(function(rows, callback){
    //turn the rows into something smartystreets can read
    var addressList = rows.map(function(d){
      return {
        street:   d[options.streetCol] || '',
        city:     d[options.cityCol] || '',
        state:    d[options.stateCol] || '',
        zipcode:  d[options.zipcodeCol] || ''
      };
    }).filter(function(d){
      if (!d.street) {
        return false;
      }
      if (!d.zipcode || (!d.city && !d.state)) {
        return false;
      }
      return true;
    });

    if (!addressList.length) {
      process.nextTick(callback);
      return;
    }

    //send the post request
    request.post({
      uri: 'https://api.smartystreets.com/street-address?'+querystring.stringify({
        'auth-id': options.authId,
        'auth-token': options.authToken
      }),
      json: addressList,
      forever: true, //use http keepAlive
      pool: pool //don't use the default connection pool (for performance)
    }, function(err, response, body){
      if (err || response.statusCode != 200) {
        if ((response && response.statusCode === 504) || (err && (err.code == 'ECONNRESET' || err.code == 'ENOTFOUND'))) {
          console.error('connection failed, retrying chunk', err || response.statusCode);
          geocoder.push([rows], function(){
            self.emit('progress', progress);
          });
{'fraction_non_alphanumeric': 0.15310427968655818, 'fraction_numerical': 0.008438818565400843, 'mean_word_length': 2.818565400843882, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4052998', 'n_tokens_mistral': 3271, 'n_tokens_neox': 3130, 'n_words': 784}