<commit_before>/*
 * dependencies.js: Common utility functions for building complete dependency trees.
 *
 * (C) 2010, Nodejitsu Inc.
 *
 */

var path = require('path'),
    semver = require('semver'),
    utile = require('utile'),
    wtfos = require('wtfos'),
    async = utile.async,
    validate = require('./validate');

//
// ### function dependencies (options, callback)
// #### @options {Object} {Options for calculating dependencies}
// ####   @options.systems {Array|Object} List of systems to create dependency tree for.
// ####   @options.client  {Client}       API Client for fetching systems.
// ####   @options.os      {string}       Target OS to calculate dependencies for.
// ####   @options.all     {Object}       Set of all dependencies resolved.
// #### @callback {function} Continuation to respond to when complete.
//
// Creates a dependency tree for the specified `systems`.
//
var dependencies = module.exports = function (options, callback) {
  var systems = options.systems,
      client  = options.client,
      os      = options.os,
      all     = options.all || {},
      tree    = {};

  //
  // Helper function which builds the subtree for
  // `name` and inserts it into the parent `tree`.
  //
  function updateTree(parts, next) {
    var name    = parts[0],
        version = parts[1];

    //
    // If the system `name` has already been fetched
    // then just set it in the tree. 
    //
    if (all[name]) {
      tree[name] = all[name];
      return next();
    }
    
    client.get(name, function (err, system) {
      if (err) {
        return next(new Error('Error fetching ' + name + ': ' + err.message.replace(/Error /, '')));
      }

      var versions = Object.keys(system.versions),
          required = version || "*",
          invalid,
          osdeps;

      version = semver.maxSatisfying(
        versions,
        version || system.version
      ) || version;

      if (!system.versions[version]) {
        return next(new Error('Could not resolve dependency: ' + [name, version].join('@')));
      }

      system              = system.versions[version],
      system.version      = version;
      system.required     = required;
      system.runlist      = system.runlist || [];
      system.dependencies = utile.mixin(
        system.dependencies || {},
        osDependencies(system, os)
      );

      if ((!system.runlist || !system.runlist.length) && system.dependencies) {
        system.runlist = Object.keys(system.dependencies);
      }

      invalid = validate(system, os);
      if (invalid) {
        return next(invalid);
      }
      
      //
      // Set the system in the tree and the
      // set of all systems we've fetched.
      //
      tree[name] = all[name] = system;

      if ((!system.dependencies || !Object.keys(system.dependencies).length)
          && (!system.remoteDependencies || !Object.keys(system.remoteDependencies).length)
          && (!system.os || !system.os[os])) {
        return next();
      }

      async.parallel({
        dependencies: async.apply(dependencies, {
          systems: system.dependencies,
          client: client,
          all: all,
          os: os
        }),
        remoteDependencies: function remoteDeps(next) {
          if (!system.remoteDependencies) {
            return next();
          }
          
          dependencies({
            systems: system.remoteDependencies,
            client: client,
            all: all
          }, next);
        }
      }, function (err, subtrees) {
        if (err) {
          return next(err);
        }

        ['dependencies', 'remoteDependencies'].forEach(function (key) {
          if (subtrees[key]) {
            tree[name][key] = all[name][key] = subtrees[key];
          }
        });

        next();
      });
    });
  }

  async.forEach(dependencies.systemNames(systems), updateTree, function (err) {<commit_msg>[api] Add remote dependencies to roles<commit_after>/*
 * dependencies.js: Common utility functions for building complete dependency trees.
 *
 * (C) 2010, Nodejitsu Inc.
 *
 */

var path = require('path'),
    semver = require('semver'),
    utile = require('utile'),
    wtfos = require('wtfos'),
    async = utile.async,
    validate = require('./validate');

//
// ### function dependencies (options, callback)
// #### @options {Object} {Options for calculating dependencies}
// ####   @options.systems {Array|Object} List of systems to create dependency tree for.
// ####   @options.client  {Client}       API Client for fetching systems.
// ####   @options.os      {string}       Target OS to calculate dependencies for.
// ####   @options.all     {Object}       Set of all dependencies resolved.
// #### @callback {function} Continuation to respond to when complete.
//
// Creates a dependency tree for the specified `systems`.
//
var dependencies = module.exports = function (options, callback) {
  var systems          = options.systems,
      depsClient       = options.dependenciesClient,
      remoteDepsClient = options.remoteDependenciesClient || depsClient,
      os               = options.os,
      all              = options.all || {},
      type             = options.type || ['remoteDependencies', 'dependencies'],
      tree             = {};

  if (!depsClient) {
    depsClient = remoteDepsClient = options.client;
  }

  //
  // Helper function which builds the subtree for
  // `name` and inserts it into the parent `tree`.
  //
  function updateTree(parts, next) {
    var name    = parts[0],
        version = parts[1];

    //
    // If the system `name` has already been fetched
    // then just set it in the tree. 
    //
    if (all[name]) {
      tree[name] = all[name];
      return next();
    }
    
    depsClient.get(name, function (err, system) {
      if (err) {
        return next(new Error('Error fetching ' + name + ': ' + err.message.replace(/Error /, '')));
      }

      var versions = Object.keys(system.versions),
          required = version || "*",
          invalid,
          osdeps;

      version = semver.maxSatisfying(
        versions,
        version || system.version
      ) || version;

      if (!system.versions[version]) {
        return next(new Error('Could not resolve dependency: ' + [name, version].join('@')));
      }

      system              = system.versions[version],
      system.version      = version;
      system.required     = required;
      system.runlist      = system.runlist || [];
      system.dependencies = utile.mixin(
        system.dependencies || {},
        osDependencies(system, os)
      );

      if ((!system.runlist || !system.runlist.length) && system.dependencies) {
        system.runlist = Object.keys(system.dependencies);
      }

      invalid = validate(system, os);
      if (invalid) {
        return next(invalid);
      }
      
      //
      // Set the system in the tree and the
      // set of all systems we've fetched.
      //
      tree[name] = all[name] = system;

      if ((!system.dependencies || !Object.keys(system.dependencies).length)
          && (!system.remoteDependencies || !Object.keys(system.remoteDependencies).length)
          && (!system.os || !system.os[os])) {
        return next();
      }

      async.parallel({
        dependencies: function deps(next) {
          if (!system.dependencies || type.indexOf('dependencies') === -1) {
            return next();
          }

          dependencies({
            systems: system.dependencies,
            client: depsClient,
            all: all,
            type: type,
            os: os
          }, next);
        },
        remoteDependencies: function remoteDeps(next) {
          if (!system.remoteDependencies || type.indexOf('remoteDependencies')) {
            return next();
          }

          dependencies({
            systems: system.remoteDependencies,
            client: remoteDepsClient,
            type: type,
            all: all
          }, next);
        }
      }, function (err, subtrees) {
        if (err) {
          return next(err);
        }

        ['dependencies', 'remoteDependencies'].forEach(function (key) {
          if (subtrees[key]) {
            tree[name][key] = all[name][key] = subtrees[key];
          }
        });

        next();
      });
    });
  }

  async.forEach(dependencies.systemNames(systems), updateTree, function (err) {
{'fraction_non_alphanumeric': 0.13713809126653162, 'fraction_numerical': 0.0015489098057905398, 'mean_word_length': 2.0118406889128093, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2314648', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2248, 'n_words': 587}