<commit_before>var util = require('util');
var npm = require('npm');
var _ = require('lodash');
var async = require('async');
var path = require('path');

require('es6-collections');

var cacheLocation = '/Users/jryans/projects/core-score/npmcache'; 

var npmConfig = {
  cache: cacheLocation
};

var coreModules = require('repl')._builtinLibs;

exports.scoreModules = scoreModules;

function scoreModules(cb) {
  npm.load(npmConfig, function(err) {
    /*npm.commands.search('mkdirp', true, function(err, data) {
      if (err) {
        return cb(err);
      }

      // Object.keys(data).slice(0, 1).forEach(grabPackage);
    });*/

    new ScoringModule('mkdirp').score(cb);
  });
}

function ScoringModule(moduleName) {
  var self = this;

  self.moduleName = moduleName;
  self.filesSeen = new Set();
  self.coreCount = new Map();
  
  coreModules.forEach(function(coreMod) {
    self.coreCount.set(coreMod, 0);
  });
}

ScoringModule.prototype.score = function(cb) {
  var self = this;
  async.waterfall([
    self.grabPackage.bind(self),
    self.findMain.bind(self)
  ], cb);
};

ScoringModule.prototype.seen = function(modRelPath) {
  return this.filesSeen.has(modRelPath);
};

ScoringModule.prototype.grabPackage = function(cb) {
  var self = this;
  npm.commands.info([self.moduleName, 'dist.tarball'], true, function(err, info) {
    if (err) {
      return cb(err);
    }

    var tarUrl = _.values(info)[0]['dist.tarball'];

    npm.commands.cache.add(tarUrl, function(err, json) {
      if (err) {
        return cb(err);
      }

      var modulePath = path.resolve(cacheLocation, self.moduleName, _.keys(info)[0], 'package');

      cb(null, json, modulePath);
    });
  });
};

ScoringModule.prototype.findMain = function(json, modulePath, cb) {
  var self = this;

  var mainFile;

  cb(new Error("No main file found for " + self.moduleName));
};<commit_msg>Add main file search<commit_after>var util = require('util');
var npm = require('npm');
var _ = require('lodash');
var async = require('async');
var path = require('path');
var fs = require('fs');

require('es6-collections');

var cacheLocation = '/Users/jryans/projects/core-score/npmcache'; 

var npmConfig = {
  cache: cacheLocation
};

var coreModules = require('repl')._builtinLibs;

exports.scoreModules = scoreModules;

function scoreModules(cb) {
  npm.load(npmConfig, function(err) {
    /*npm.commands.search('mkdirp', true, function(err, data) {
      if (err) {
        return cb(err);
      }

      // Object.keys(data).slice(0, 1).forEach(grabPackage);
    });*/

    new ScoringModule('mkdirp').score(cb);
  });
}

function ScoringModule(moduleName) {
  var self = this;

  self.moduleName = moduleName;
  self.filesSeen = new Set();
  self.coreCount = new Map();
  
  coreModules.forEach(function(coreMod) {
    self.coreCount.set(coreMod, 0);
  });
}

ScoringModule.prototype.score = function(cb) {
  var self = this;
  async.waterfall([
    self.grabPackage.bind(self),
    self.findMain.bind(self)
  ], cb);
};

ScoringModule.prototype.seen = function(modRelPath) {
  return this.filesSeen.has(modRelPath);
};

ScoringModule.prototype.grabPackage = function(cb) {
  var self = this;
  npm.commands.info([self.moduleName, 'dist.tarball'], true, function(err, info) {
    if (err) {
      return cb(err);
    }

    var tarUrl = _.values(info)[0]['dist.tarball'];

    npm.commands.cache.add(tarUrl, function(err, json) {
      if (err) {
        return cb(err);
      }

      var modulePath = path.resolve(cacheLocation, self.moduleName, _.keys(info)[0], 'package');

      cb(null, json, modulePath);
    });
  });
};

ScoringModule.prototype.findMain = function(json, modulePath, cb) {
  var self = this;

  var mainFilePath = _.find([
    (json && json.main) ? json.main : null,
    (json && json.main) ? json.main + '.js' : null,
    './index.js'
  ], function(testPath) {
    if (!testPath) {
      return false;
    }
 
    testPath = path.resolve(modulePath, testPath);
    if (fs.existsSync(testPath)) {
      return path.relative(modulePath, testPath);
    }

    return false;
  });

  util.debug(mainFilePath);

  cb(new Error("No main file found for " + self.moduleName));
};
{'fraction_non_alphanumeric': 0.16320191158900837, 'fraction_numerical': 0.002867383512544803, 'mean_word_length': 3.5303030303030303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45290', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1412, 'n_words': 275}