<commit_before><commit_msg>Add index<commit_after>var jsdom = require('jsdom'),
  request = require('request'),
  url = require('url'),
  npm = require("npm"),
  redis = require("redis");

var client = redis.createClient();

var configObject = {
  "dev": false,
  "loglevel": "error"
};

getData(0);

function getData(j) {
  request({
    uri: 'https://npmjs.org/browse/all/' + j
  }, function (err, response, body) {

    if (err && response.statusCode !== 200) {
      console.log('Request error.');
    }

    npm.load(configObject, function (er, npm) {
      jsdom.env({
        html: body,
        scripts: ['http://code.jquery.com/jquery-1.6.min.js']
      }, function (err, window) {
        var $ = window.jQuery;
        var $rows = $('body').find('.row');

        if ($rows.length > 1) {
          $rows.each(function (i, item) {
            var $a = $(item).find('a');
            var module = $a.html();

            crawled(module);
          });
          getData(j + 1);
        }
      });
    });
  });
}

function crawled(module) {
  client.sadd("crawled", module, function (err, idd) {
    if (idd == 1) {
      npm.commands.view([module], true, function (er, data2) {
        if (data2 != undefined) {
          var mm = data2[Object.keys(data2)[0]];
          if (mm != undefined) {
            var deps = mm.dependencies;
            var module = mm.name;
            if (deps != undefined && Object.keys(deps).length > 0) {
              for(var i = 0; i < Object.keys(deps).length; i++) {
                var lib = Object.keys(deps)[i];
                console.log(lib + " <- " + module);
                client.sadd(lib, module);
              }
            }
          }
        }
      });
    }
  });
}
{'fraction_non_alphanumeric': 0.18024263431542462, 'fraction_numerical': 0.00924321201617562, 'mean_word_length': 1.8533772652388798, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1280899', 'n_tokens_mistral': 583, 'n_tokens_neox': 557, 'n_words': 118}