<commit_before>#! /usr/bin/env node

'use strict';

const Bottleneck = require("bottleneck");
const request = require('request-promise');
const mp = require('mongodb-promise');

var limiter = new Bottleneck(1000, 5);

const XKCD = 'http://xkcd.com/';
const JSON_URL = 'info.0.json';
var promiseArr = [];

function _insert(json) {
  mp.MongoClient.connect('mongodb://heroku_97mjvv9b:<EMAIL>:45997/heroku_97mjvv9b')
    .then(db => {
      return db.collection('records')
        .then(col => col.insert(json))
        .then(result => {
          console.log(result);
          db.close()
            .then(console.log('success'));
        });
    })
    .fail(err => console.log(err));
}

function _getLatest(json) {
  _insert(json);
  return json.num;
}

function _createCache(num) {
  let promise = _fetchJSON(num)
    .then(json => {
      _insert(json);
    });
  promiseArr.push(promise);
  return promise;
}

function _getAll(latest) {
  for (var i = 1; i < latest; i++) {
    limiter.schedule(_createCache, i)
      .catch(err => {
        console.log(err.message);
      });
  }

  return Promise.all(promiseArr);
}

function _fetchJSON(num) {
  var options = {
    url: `${XKCD}/${num || ''}/${JSON_URL}`,
    json: true
  }

  return request(options);
}

_fetchJSON()
  .then(_getLatest)
  .then(_getAll)
  .then(arr => console.log(arr))
  .catch(err => {
    console.log(data);
    console.error(err.message);
  });<commit_msg>Update xkcd-dump<commit_after>#! /usr/bin/env node

'use strict';

const Bottleneck = require("bottleneck");
const request = require('request-promise');
const mp = require('mongodb-promise');

var limiter = new Bottleneck(100000, 10);

const XKCD = 'http://xkcd.com/';
const JSON_URL = 'info.0.json';
var promiseArr = [];
var db;

mp.MongoClient.connect('mongodb://heroku_97mjvv9b:l7qh0eg<EMAIL>:45997/heroku_97mjvv9b')
  .then(database => {
    db = database;

    _fetchJSON()
      .then(_getLatest)
      .then(_getAll)
      .then(arr => console.log(arr))
      .catch(err => {
        console.log(data);
        console.error(err.message);
      });
  }).fail(err => console.log(err));

function _checkIfExists(json) {
  return db.collection('records')
    .then(col => col.find({
      'num': json.num
    }).toArray())
    .then(items => {
      if (items.length) {
        return Promise.reject('already exists');
      } else {
        return Promise.resolve(json);
      }
    });
}

function _insert(json) {
  console.log(json.num);
  return db.collection('records')
    .then(col => col.insert(json))
    .then(result => {
      console.log(result);
      db.close()
        .then(console.log('success'));
    });
}

function _getLatest(json) {
  _insert(json);
  return json.num;
}

function _createCache(num) {
  return _fetchJSON(num).then(_checkIfExists).then(_insert);
}

function _getAll(latest) {
  for (var i = 1; i < latest; i++) {
    limiter.schedule(_createCache, i)
      .catch(function (err) {
        console.log(`${this} : ${err.message}`);
      }.bind(i));
  }

  return Promise.all(promiseArr);
}

function _fetchJSON(num) {
  var options = {
    url: `${XKCD}/${num || ''}/${JSON_URL}`,
    json: true
  }

  return request(options);
}
{'fraction_non_alphanumeric': 0.17903930131004367, 'fraction_numerical': 0.012788521522145976, 'mean_word_length': 3.0697969543147208, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009830', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1143, 'n_words': 201}