<commit_before>var path = require('path');
var fs = require('fs');
var IsThere = require('is-there');
var objectAssign = require('object-assign');

var MODULE_DIR = 'node_modules';

var moduleReader = {
  readPackageJson: function(mod) {
    var pathName = path.join(this.context, MODULE_DIR, mod, 'package.json');
    var file = fs.readFileSync(pathName);
    return JSON.parse(file);
  },
  getModuleInfo: function(mod) {
    var packagejson = this.readPackageJson(mod);
    return packagejson;
  },
  parseModuleInfo: function(mod) {
    var packagejson = this.moduleCache[mod];
    return {
      name: mod,
      url: packagejson.repository && packagejson.repository.url,
      version: packagejson.version,
      license: packagejson.license,
      licenseText: this.getLicenseText(mod, packagejson.license)
    };
  },
  findPackageName: function(jsFilePath) {
    return jsFilePath
      .replace(path.join(this.context, MODULE_DIR) + path.sep, '')
      .split(path.sep)
      .filter(function(value, index, arr) {
        return value.charAt(0) === '@'
          || (index === 0 && value.charAt(0) !== '@')
          || (index > 0 && arr[index-1].charAt(0) === '@');
      })
      .join('/');
  },
  writeModuleInfo: function() {
    this.modules = Object.keys(this.moduleMap)
      .filter(function(mod) {
        if(mod.trim() === '') {
          return false;
        }
        var moduleInfo = this.getModuleInfo(mod);
        var isMatching = this.pattern.test(moduleInfo.license) || !moduleInfo.license && this.includeUndefined;
        if (isMatching) {
          this.moduleCache[mod] = moduleInfo;
        }
        return isMatching;
      }.bind(this))
      .map(this.parseModuleInfo.bind(this));
  },
  gatherModuleInfo: function(moduleStats) {
    var moduleMap = {};
    moduleStats
      .filter(function(mod) {
        return !!mod.resource;
      })
      .forEach(function(mod) {
        var moduleName = this.findPackageName(mod.resource);
        moduleMap[moduleName] = {};
      }.bind(this));
    this.moduleMap = moduleMap;
    this.writeModuleInfo();
  }
};

var licenseReader = {
  getLicenseText: function(mod, license) {
    var licenseText = '';
    var file =
      (this.licenseOverrides[mod] && IsThere(this.licenseOverrides[mod])) ?
      this.licenseOverrides[mod] : this.findLicenseFile(mod);<commit_msg>fix: ignore files outside of node_modules<commit_after>var path = require('path');
var fs = require('fs');
var IsThere = require('is-there');
var objectAssign = require('object-assign');

var MODULE_DIR = 'node_modules';

var moduleReader = {
  isFromNodeModules: function(mod) {
    var jsFilePath = mod.resource;
    var modulePrefix = path.join(this.context, MODULE_DIR);
    return !!jsFilePath && jsFilePath.startsWith(modulePrefix);
  },
  readPackageJson: function(mod) {
    var pathName = path.join(this.context, MODULE_DIR, mod, 'package.json');
    var file = fs.readFileSync(pathName);
    return JSON.parse(file);
  },
  getModuleInfo: function(mod) {
    var packagejson = this.readPackageJson(mod);
    return packagejson;
  },
  parseModuleInfo: function(mod) {
    var packagejson = this.moduleCache[mod];
    return {
      name: mod,
      url: packagejson.repository && packagejson.repository.url,
      version: packagejson.version,
      license: packagejson.license,
      licenseText: this.getLicenseText(mod, packagejson.license)
    };
  },
  findPackageName: function(jsFilePath) {
    return jsFilePath
      .replace(path.join(this.context, MODULE_DIR) + path.sep, '')
      .split(path.sep)
      .filter(function(value, index, arr) {
        return value.charAt(0) === '@'
          || (index === 0 && value.charAt(0) !== '@')
          || (index > 0 && arr[index-1].charAt(0) === '@');
      })
      .join('/');
  },
  writeModuleInfo: function() {
    this.modules = Object.keys(this.moduleMap)
      .filter(function(mod) {
        if(mod.trim() === '') {
          return false;
        }
        var moduleInfo = this.getModuleInfo(mod);
        var isMatching = this.pattern.test(moduleInfo.license) || !moduleInfo.license && this.includeUndefined;
        if (isMatching) {
          this.moduleCache[mod] = moduleInfo;
        }
        return isMatching;
      }.bind(this))
      .map(this.parseModuleInfo.bind(this));
  },
  gatherModuleInfo: function(moduleStats) {
    var moduleMap = {};
    moduleStats
      .filter(this.isFromNodeModules.bind(this))
      .forEach(function(mod) {
        var moduleName = this.findPackageName(mod.resource);
        moduleMap[moduleName] = {};
      }.bind(this));
    this.moduleMap = moduleMap;
    this.writeModuleInfo();
  }
};

var licenseReader = {
  getLicenseText: function(mod, license) {
    var licenseText = '';
    var file =
      (this.licenseOverrides[mod] && IsThere(this.licenseOverrides[mod])) ?
      this.licenseOverrides[mod] : this.findLicenseFile(mod);
{'fraction_non_alphanumeric': 0.14765784114052954, 'fraction_numerical': 0.002443991853360489, 'mean_word_length': 3.045304777594728, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '543691', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1520, 'n_words': 272}