<commit_before>var fs = require('fs');
var path = require('path');
var _ = require('underscore');
var packageClient = require('./package-client.js');
var watch = require('./watch.js');
var archinfo = require('./archinfo.js');
var isopack = require('./isopack.js');
var compiler = require('./compiler.js');
var buildmessage = require('./buildmessage.js');
var tropohouse = require('./tropohouse.js');
var files = require('./files.js');
var utils = require('./utils.js');
var catalog = require('./catalog.js');
var packageCache = require('./package-cache.js');
var PackageSource = require('./package-source.js');
var VersionParser = require('./package-version-parser.js');

// LocalCatalog represents the packages located into an application folder
// A default instance of this catalog is created in catalog.js
var LocalCatalog = function (options) {
  var self = this;

  // Package server data. Mostly arrays of objects.
  self.packages = null;
  self.versions = null;  // package name -> version -> object
  self.builds = null;

  // We use the initialization design pattern because it makes it easier to use
  // both of our catalogs as singletons.
  self.initialized = false;
  self.containingCatalog = options ? options.containingCatalog : self;

    // Local directories to search for package source trees
  self.localPackageDirs = null;

  // Packagedirs specified by addLocalPackage: added explicitly through a
  // directory. We mainly use this to allow the user to run test-packages against a
  // package in a specific directory.
  self.localPackages = [];

  // All packages found either by localPackageDirs or localPackages. There is a
  // hierarghy of packages, as detailed below and there can only be one local
  // version of a package at a time. This refers to the package by the specific
  // package directory that we need to process.
  self.effectiveLocalPackages = [];

   // Each catalog needs its own package cache.
  self.packageCache = new packageCache.PackageCache(self.containingCatalog);

  self.packageSources = null;
  self.built = null;
};

_.extend(LocalCatalog.prototype, {
  toString: function () {
    var self = this;
    return "LocalCatalog [localPackageDirs=" + self.localPackageDirs + "]";
  },

  // Initialize the Catalog. This must be called before any other
  // Catalog function.

  // options:
  //  - localPackageDirs: an array of paths on local disk, that
  //    contain subdirectories, that each contain a source tree for a
  //    package that should override the packages on the package
  //    server. For example, if there is a package 'foo' that we find
  //    through localPackageDirs, then we will ignore all versions of
  //    'foo' that we find through the package server. Directories
  //    that don't exist (or paths that aren't directories) will be
  //    silently ignored.
  initialize: function (options) {
    var self = this;
    buildmessage.assertInCapture();

    options = options || {};

    // initializing this here to make it clear that this exists and we have
    // access to it -- a map of names of local packages to their package
    // sources. We call upon this when we compile the package.
    self.packageSources = {};

    // At this point, effectiveLocalPackageDirs is just the local package
    // directories, since we haven't had a chance to add any other local
    // packages. Nonetheless, let's set those.
    self.localPackageDirs =
      _.filter(options.localPackageDirs || [], utils.isDirectory);

    self.refresh({initializing: true});
  },

  // Set all the collections to their initial values, which are mostly
  // blank. This does not set self.initialized -- do that manually in the child
  // class when applicable.
  reset: function () {
    var self = this;

    // Initialize everything to its default version.
    self.packages = [];
    self.versions = {};
    self.builds = [];

    self.packageSources = {};
    self.built = {};
  },

  // Throw if the catalog's self.initialized value has not been set to true.
  _requireInitialized: function () {
    var self = this;

    if (! self.initialized)
      throw new Error("catalog not initialized yet?");
  },

  // Return an array with the names of all of the packages that we
  // know about, in no particular order.
  getAllPackageNames: function () {
    var self = this;
    self._requireInitialized();

    return _.pluck(self.packages, 'name');
  },

  // Returns general (non-version-specific) information about a
  // package, or null if there is no such package.
  getPackage: function (name, options) {
    var self = this;
    buildmessage.assertInCapture();
    self._requireInitialized();
    options = options || {};

    return _.findWhere(self.packages, { name: name });
  },

  // Given a package, returns an array of the versions available for
  // this package (for any architecture), sorted from oldest to newest
  // (according to the version string, not according to their
  // publication date). Returns the empty array if the package doesn't
  // exist or doesn't have any versions.
  //
  // (XXX: If local catalog is just the local overrides, wouldn't this always
  // just return one record?)
  getSortedVersions: function (name) {
    var self = this;
    self._requireInitialized();
    if (!_.has(self.versions, name)) {
      return [];
    }
    var ret = _.keys(self.versions[name]);
    ret.sort(VersionParser.compare);
    return ret;
  },

  // Return information about a particular version of a package, or
  // null if there is no such package or version.
  getVersion: function (name, version) {
    var self = this;
    self._requireInitialized();
    buildmessage.assertInCapture();

    var lookupVersion = function () {
      return _.has(self.versions, name) &&
        _.has(self.versions[name], version) &&
        self.versions[name][version];
    };

    // The catalog doesn't understand buildID versions and doesn't know about
    // them. Depending on when we build them, we can refer to local packages as
    // 1.0.0+local or 1.0.0+[buildId]. Luckily, we know which packages are
    // local, so just look those up by their local version instead.
    // XXX ideally we'd only have isLocalPackage in the complete catalog and
    //     have CompleteCatalog override getVersion, but other things want
    //     to call isLocalPackage, eg maybeDownloadPackageForArchitectures
    //     which has the official package when running make-bootstrap-tarballs
    if (self.isLocalPackage(name)) {
      version = self._getLocalVersion(version);
      // No need to refresh here: if we can't find the local version, refreshing
      // isn't going to help!
      return lookupVersion() || null;
    }

    return lookupVersion() || null;
  },

  // As getVersion, but returns info on the latest version of the
  // package, or null if the package doesn't exist or has no versions.
  getLatestVersion: function (name) {
    var self = this;

    var versions = self.getSortedVersions(name);
    versions.reverse();
    return self.getVersion(name, versions[0]);
  },

  // Overridden by CompleteCatalog.
  // XXX this is kinda sketchy, maybe callers should only call this
  //     on the CompleteCatalog?
  isLocalPackage: function () {
    return false;
  },

  // If this package has any builds at this version, return an array of builds
  // which cover all of the required arches, or null if it is impossible to
  // cover them all (or if the version does not exist).
  getBuildsForArches: function (name, version, arches) {
    var self = this;
    self._requireInitialized();
    buildmessage.assertInCapture();

    var versionInfo = self.getVersion(name, version);
    if (! versionInfo)
      return null;

    // XXX if we have a choice between os and os.mac, this returns a random one.
    //     so in practice we don't really support "maybe-platform-specific"
    //     packages

    var allBuilds = _.where(self.builds, { versionId: versionInfo._id });
    var solution = null;
    utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) {
      // This build subset works if for all the arches we need, at least one
      // build in the subset satisfies it. It is guaranteed to be minimal,
      // because we look at subsets in increasing order of size.
      var satisfied = _.all(arches, function (neededArch) {
        return _.any(buildSubset, function (build) {
          var buildArches = build.buildArchitectures.split('+');
          return !!archinfo.mostSpecificMatch(neededArch, buildArches);
        });
      });
      if (satisfied) {<commit_msg>Remove duplicate method definition<commit_after>var fs = require('fs');
var path = require('path');
var _ = require('underscore');
var packageClient = require('./package-client.js');
var watch = require('./watch.js');
var archinfo = require('./archinfo.js');
var isopack = require('./isopack.js');
var compiler = require('./compiler.js');
var buildmessage = require('./buildmessage.js');
var tropohouse = require('./tropohouse.js');
var files = require('./files.js');
var utils = require('./utils.js');
var catalog = require('./catalog.js');
var packageCache = require('./package-cache.js');
var PackageSource = require('./package-source.js');
var VersionParser = require('./package-version-parser.js');

// LocalCatalog represents the packages located into an application folder
// A default instance of this catalog is created in catalog.js
var LocalCatalog = function (options) {
  var self = this;

  // Package server data. Mostly arrays of objects.
  self.packages = null;
  self.versions = null;  // package name -> version -> object
  self.builds = null;

  // We use the initialization design pattern because it makes it easier to use
  // both of our catalogs as singletons.
  self.initialized = false;
  self.containingCatalog = options ? options.containingCatalog : self;

    // Local directories to search for package source trees
  self.localPackageDirs = null;

  // Packagedirs specified by addLocalPackage: added explicitly through a
  // directory. We mainly use this to allow the user to run test-packages against a
  // package in a specific directory.
  self.localPackages = [];

  // All packages found either by localPackageDirs or localPackages. There is a
  // hierarghy of packages, as detailed below and there can only be one local
  // version of a package at a time. This refers to the package by the specific
  // package directory that we need to process.
  self.effectiveLocalPackages = [];

   // Each catalog needs its own package cache.
  self.packageCache = new packageCache.PackageCache(self.containingCatalog);

  self.packageSources = null;
  self.built = null;
};

_.extend(LocalCatalog.prototype, {
  toString: function () {
    var self = this;
    return "LocalCatalog [localPackageDirs=" + self.localPackageDirs + "]";
  },

  // Initialize the Catalog. This must be called before any other
  // Catalog function.

  // options:
  //  - localPackageDirs: an array of paths on local disk, that
  //    contain subdirectories, that each contain a source tree for a
  //    package that should override the packages on the package
  //    server. For example, if there is a package 'foo' that we find
  //    through localPackageDirs, then we will ignore all versions of
  //    'foo' that we find through the package server. Directories
  //    that don't exist (or paths that aren't directories) will be
  //    silently ignored.
  initialize: function (options) {
    var self = this;
    buildmessage.assertInCapture();

    options = options || {};

    // initializing this here to make it clear that this exists and we have
    // access to it -- a map of names of local packages to their package
    // sources. We call upon this when we compile the package.
    self.packageSources = {};

    // At this point, effectiveLocalPackageDirs is just the local package
    // directories, since we haven't had a chance to add any other local
    // packages. Nonetheless, let's set those.
    self.localPackageDirs =
      _.filter(options.localPackageDirs || [], utils.isDirectory);

    self.refresh({initializing: true});
  },

  // Set all the collections to their initial values, which are mostly
  // blank. This does not set self.initialized -- do that manually in the child
  // class when applicable.
  reset: function () {
    var self = this;

    // Initialize everything to its default version.
    self.packages = [];
    self.versions = {};
    self.builds = [];

    self.packageSources = {};
    self.built = {};
  },

  // Throw if the catalog's self.initialized value has not been set to true.
  _requireInitialized: function () {
    var self = this;

    if (! self.initialized)
      throw new Error("catalog not initialized yet?");
  },

  // Return an array with the names of all of the packages that we
  // know about, in no particular order.
  getAllPackageNames: function () {
    var self = this;
    self._requireInitialized();

    return _.pluck(self.packages, 'name');
  },

  // Returns general (non-version-specific) information about a
  // package, or null if there is no such package.
  getPackage: function (name, options) {
    var self = this;
    buildmessage.assertInCapture();
    self._requireInitialized();
    options = options || {};

    return _.findWhere(self.packages, { name: name });
  },

  // Given a package, returns an array of the versions available for
  // this package (for any architecture), sorted from oldest to newest
  // (according to the version string, not according to their
  // publication date). Returns the empty array if the package doesn't
  // exist or doesn't have any versions.
  //
  // (XXX: If local catalog is just the local overrides, wouldn't this always
  // just return one record?)
  getSortedVersions: function (name) {
    var self = this;
    self._requireInitialized();
    if (!_.has(self.versions, name)) {
      return [];
    }
    var ret = _.keys(self.versions[name]);
    ret.sort(VersionParser.compare);
    return ret;
  },

  // Return information about a particular version of a package, or
  // null if there is no such package or version.
  getVersion: function (name, version) {
    var self = this;
    self._requireInitialized();
    buildmessage.assertInCapture();

    var lookupVersion = function () {
      return _.has(self.versions, name) &&
        _.has(self.versions[name], version) &&
        self.versions[name][version];
    };

    // The catalog doesn't understand buildID versions and doesn't know about
    // them. Depending on when we build them, we can refer to local packages as
    // 1.0.0+local or 1.0.0+[buildId]. Luckily, we know which packages are
    // local, so just look those up by their local version instead.
    // XXX ideally we'd only have isLocalPackage in the complete catalog and
    //     have CompleteCatalog override getVersion, but other things want
    //     to call isLocalPackage, eg maybeDownloadPackageForArchitectures
    //     which has the official package when running make-bootstrap-tarballs
    if (self.isLocalPackage(name)) {
      version = self._getLocalVersion(version);
      // No need to refresh here: if we can't find the local version, refreshing
      // isn't going to help!
      return lookupVersion() || null;
    }

    return lookupVersion() || null;
  },

  // As getVersion, but returns info on the latest version of the
  // package, or null if the package doesn't exist or has no versions.
  getLatestVersion: function (name) {
    var self = this;

    var versions = self.getSortedVersions(name);
    versions.reverse();
    return self.getVersion(name, versions[0]);
  },

  // If this package has any builds at this version, return an array of builds
  // which cover all of the required arches, or null if it is impossible to
  // cover them all (or if the version does not exist).
  getBuildsForArches: function (name, version, arches) {
    var self = this;
    self._requireInitialized();
    buildmessage.assertInCapture();

    var versionInfo = self.getVersion(name, version);
    if (! versionInfo)
      return null;

    // XXX if we have a choice between os and os.mac, this returns a random one.
    //     so in practice we don't really support "maybe-platform-specific"
    //     packages

    var allBuilds = _.where(self.builds, { versionId: versionInfo._id });
    var solution = null;
    utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) {
      // This build subset works if for all the arches we need, at least one
      // build in the subset satisfies it. It is guaranteed to be minimal,
      // because we look at subsets in increasing order of size.
      var satisfied = _.all(arches, function (neededArch) {
        return _.any(buildSubset, function (build) {
          var buildArches = build.buildArchitectures.split('+');
          return !!archinfo.mostSpecificMatch(neededArch, buildArches);
        });
      });
      if (satisfied) {
{'fraction_non_alphanumeric': 0.10123311802701115, 'fraction_numerical': 0.0008220786846741045, 'mean_word_length': 3.0444075041557825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742806', 'n_tokens_mistral': 4600, 'n_tokens_neox': 4488, 'n_words': 1840}