<commit_before>module.exports = giblets;

var sander = require('sander'),
    Promise = sander.Promise,
    request = require('request-promise'),
    path = require('path');

function giblets(input, output, opts) {
  var env = opts.environment || opts.env || 'development';
  var adapt = 'adapt' in opts ? opts.adapt : true;
  var cacheDir = opts.cacheDir || path.resolve(process.cwd(), '.giblets');

  return sander.readFile(input, 'giblets.json').then(toJSON).then(function(json) {
    var deps = json[env] || {},
        results = [];

    // process giblethub
    results.push(processGibletHub(deps.giblethub || [], { base: output, adapt: adapt, cacheDir: cacheDir }));

    // process components
    results.push(processComponents(deps.component || {}, { base: output, cacheDir: cacheDir }));

    return Promise.all(results);
  }).catch(function(e) { console.log(e); console.log(e.stack); throw e; });
}

function cache(details, src, dest, cacheDir, write) {
  if (!dest) dest = src;

  var cached = path.resolve(cacheDir, details.name, details.version, dest),
      ghf = 'https://raw.githubusercontent.com/' + details.repo + '/' + details.version + '/' + details.path + src;

  // return the cached file, if it's there
  return sander.readFile(cached).then(
    function(data) {
      if (write) return sander.writeFile(write, data).then(function() { return data; });
      else return data;
    },function() { // otherwise fetch it
      return request(ghf).then(function(data) {
        return sander.writeFile(cached, data).then(function() { // and cache it
          // and write it out if requested
          if (write) return sander.writeFile(write, data).then(function() { return data; });
          else return data;
        });
      });
    }
  );
}

function processGibletHub(gh, opts) {
  var results = [],
      version, repo, name, path = '', details;

  function gibletFile(obj, dest) {
    dest.name = dest.name || obj.name;
    dest.repo = obj.repo;
    dest.path = obj.path;
    dest.version = obj.version;
    dest.base = opts.base;
    dest.adapt = 'adapt' in dest ? dest.adapt : ('adapt' in obj ? obj.adapt : opts.adapt);
    return dest;
  }

  for (var i = 0; i < gh.length && (giblet = gh[i]); i++) {
    // for repos with a giblet.json: '[&]user/repo[/path]@version'
    if (typeof giblet === 'string') {
      version = giblet.split('@');
      repo = giblet[0].split('/');
      version = giblet[1];
      details = {};

      if (repo[0][0] === '&') {
        adapt = false;
        repo[0] = repo[0].slice(1);
      }
    }

    // object description { repo, version, [ scripts, styles, files ] }
    else {
      repo = giblet.repo.split('/');
      version = giblet.version;
      details = giblet;
    }

    // module name if not already provided
    name = details.name || repo[1];

    if (repo.length > 2) path = repo.slice(2).join('/');
    repo = repo[0] + '/' + repo[1];

    details.adapt = adapt, details.base = output, details.name = name, details.repo = repo, details.version = version, details.path = path;

    // manual definition
    if (details && (details.scripts || details.styles || details.files)) {
      results.push(processGiblethub(details, opts.cacheDir));
    }

    // grab the giblet.json
    else {
      (function(details) {
        results.push(cache(details, 'giblet.json', null, opts.cacheDir)
          .then(toJSON)
          .then(function(obj) { return gibletFile(obj, details); })
          .then(function(details) { return processGithub(details, opts.cacheDir); })
        );<commit_msg>fix string descriptor and common descriptor bugs in giblethub provider<commit_after>module.exports = giblets;

var sander = require('sander'),
    Promise = sander.Promise,
    request = require('request-promise'),
    path = require('path');

function giblets(input, output, opts) {
  var env = opts.environment || opts.env || 'development';
  var adapt = 'adapt' in opts ? opts.adapt : true;
  var cacheDir = opts.cacheDir || path.resolve(process.cwd(), '.giblets');

  return sander.readFile(input, 'giblets.json').then(toJSON).then(function(json) {
    var deps = json[env] || {},
        results = [];

    // process giblethub
    results.push(processGibletHub(deps.giblethub || [], { base: output, adapt: adapt, cacheDir: cacheDir }));

    // process components
    results.push(processComponents(deps.component || {}, { base: output, cacheDir: cacheDir }));

    return Promise.all(results);
  });
}

function cache(details, src, dest, cacheDir, write) {
  if (!dest) dest = src;

  var cached = path.resolve(cacheDir, details.name, details.version, dest),
      ghf = 'https://raw.githubusercontent.com/' + details.repo + '/' + details.version + '/' + details.path + src;

  // return the cached file, if it's there
  return sander.readFile(cached).then(
    function(data) {
      if (write) return sander.writeFile(write, data).then(function() { return data; });
      else return data;
    },function() { // otherwise fetch it
      return request(ghf).then(function(data) {
        return sander.writeFile(cached, data).then(function() { // and cache it
          // and write it out if requested
          if (write) return sander.writeFile(write, data).then(function() { return data; });
          else return data;
        });
      });
    }
  );
}

function processGibletHub(gh, opts) {
  var results = [],
      version, repo, name, path = '', details;

  function gibletFile(obj, dest) {
    dest.name = dest.name || obj.name;
    dest.repo = obj.repo;
    dest.path = obj.path;
    dest.version = obj.version;
    dest.base = opts.base;
    dest.adapt = 'adapt' in dest ? dest.adapt : ('adapt' in obj ? obj.adapt : opts.adapt);
    return dest;
  }

  for (var i = 0; i < gh.length && (giblet = gh[i]); i++) {
    // for repos with a giblet.json: '[&]user/repo[/path]@version'
    if (typeof giblet === 'string') {
      giblet = giblet.split('@');
      repo = giblet[0].split('/');
      version = giblet[1];
      details = {};

      if (repo[0][0] === '&') {
        adapt = false;
        repo[0] = repo[0].slice(1);
      }
    }

    // object description { repo, version, [ scripts, styles, files ] }
    else {
      repo = giblet.repo.split('/');
      version = giblet.version;
      details = giblet;
    }

    // module name if not already provided
    name = details.name || repo[1];

    if (repo.length > 2) path = repo.slice(2).join('/');
    repo = repo[0] + '/' + repo[1];

    details.adapt = opts.adapt, details.base = opts.output, details.name = name, details.repo = repo, details.version = version, details.path = path;

    // manual definition
    if (details && (details.scripts || details.styles || details.files)) {
      results.push(processGiblethub(details, opts.cacheDir));
    }

    // grab the giblet.json
    else {
      (function(details) {
        results.push(cache(details, 'giblet.json', null, opts.cacheDir)
          .then(toJSON)
          .then(function(obj) { return gibletFile(obj, details); })
          .then(function(details) { return processGithub(details, opts.cacheDir); })
        );
{'fraction_non_alphanumeric': 0.16287878787878787, 'fraction_numerical': 0.003647586980920314, 'mean_word_length': 2.820471596998928, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2901526', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2247, 'n_words': 544}