<commit_before>'use strict';

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var _ = require('lodash');
var url = require('url');
var async = require('async');

var pass = function() {};

var jf = require('jsonfile');
jf.spaces = 0;
var dbfile = './filings.json';
var db = jf.readFileSync(dbfile);
var saveDB = _.throttle(function() {
  console.log('saving db');
  jf.writeFileSync(dbfile, db);
}, 2000);

var download = module.exports.download = function(uri, filename, callback) {
  if (fs.existsSync(filename)) {
    console.log(filename + ' exists');
    callback(null);
  }
  else {
    request(uri, function(error, response, html) {
      if (!error) {
        if (html.trim().indexOf('<!DOCTYPE') == 0) {
          console.log(filename + ' does not exist');
          callback(null);
        }
        else {
          if (firstDocument === undefined) {
            firstDocument = filename;
          }
          console.log('saving ' + filename);
          request(uri).pipe(fs.createWriteStream(filename)).on('close', function() {
            callback(null);
          });
        }
      }
      else {
        console.log('error with ' + filename);
        callback(null);
      }
    });
  }
};

var documentBaseURL = 'http://apps.fcc.gov/ecfs/document/view?id=7521104360';
var id = 7521315311;
var firstDocument;
var downloadDocument = module.exports.downloadDocument = function(id, callback) {
  var documentURL = url.resolve(documentBaseURL, url.format({
    query: {
      id: id
    }
  }));
  download(documentURL, 'documents/' + id + '.pdf', function() {
    callback(null);
  });
};

var downloadDocuments = module.exports.downloadDocuments = function(start, end, callback) {
  start = start || id;
  end = end || start + 1000000;
  console.log('downloading documents ' + start + '-' + end);
  callback = callback || function(err, results) {
    console.log('done downloading documents ' + start + '-' + end);
    console.log('first document was ' + firstDocument);
  };
  async.eachLimit(_.range(start, end), 4, downloadDocument, function() {
    callback(null);
  });
};

var filingDetailBaseURL = 'http://apps.fcc.gov/ecfs/comment/view';
var processFiling = module.exports.processFiling = function(id, callback) {
  if (db.hasOwnProperty(id)) {
    console.log(id + ' exists in db');
    downloadDocument(db[id].documentID, function() {
      callback(null);
    });
  }
  else {
    var filingURL = url.resolve(filingDetailBaseURL, url.format({
      query: {
        id: id<commit_msg>more data<commit_after>'use strict';

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var _ = require('lodash');
var url = require('url');
var async = require('async');

var pass = function() {};

var jf = require('jsonfile');
jf.spaces = 0;
var dbfile = './filings.json';
var db = jf.readFileSync(dbfile);
var saveDB = _.throttle(function() {
  console.log('saving db');
  jf.writeFileSync(dbfile, db);
}, 2000);

var download = module.exports.download = function(uri, filename, callback) {
  if (fs.existsSync(filename)) {
    console.log(filename + ' exists');
    callback(null);
  }
  else {
    request(uri, function(error, response, html) {
      if (!error) {
        if (html.trim().indexOf('<!DOCTYPE') == 0) {
          console.log(filename + ' does not exist');
          callback(null);
        }
        else {
          if (firstDocument === undefined) {
            firstDocument = filename;
          }
          console.log('saving ' + filename);
          request(uri).pipe(fs.createWriteStream(filename)).on('close', function() {
            callback(null);
          });
        }
      }
      else {
        console.log('error with ' + filename);
        callback(null);
      }
    });
  }
};

var documentBaseURL = 'http://apps.fcc.gov/ecfs/document/view?id=7521104360';
var id = 7521487196;
var firstDocument;
var downloadDocument = module.exports.downloadDocument = function(id, callback) {
  var documentURL = url.resolve(documentBaseURL, url.format({
    query: {
      id: id
    }
  }));
  download(documentURL, 'documents/' + id + '.pdf', function() {
    callback(null);
  });
};

var downloadDocuments = module.exports.downloadDocuments = function(start, end, callback) {
  start = start || id;
  end = end || start + 1000000;
  console.log('downloading documents ' + start + '-' + end);
  callback = callback || function(err, results) {
    console.log('done downloading documents ' + start + '-' + end);
    console.log('first document was ' + firstDocument);
  };
  async.eachLimit(_.range(start, end), 10, downloadDocument, function() {
    callback(null);
  });
};

var filingDetailBaseURL = 'http://apps.fcc.gov/ecfs/comment/view';
var processFiling = module.exports.processFiling = function(id, callback) {
  if (db.hasOwnProperty(id)) {
    console.log(id + ' exists in db');
    downloadDocument(db[id].documentID, function() {
      callback(null);
    });
  }
  else {
    var filingURL = url.resolve(filingDetailBaseURL, url.format({
      query: {
        id: id
{'fraction_non_alphanumeric': 0.1612840086122529, 'fraction_numerical': 0.013505578391074574, 'mean_word_length': 3.094551282051282, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '163910', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1563, 'n_words': 340}