<commit_before>var Promise = require("nodegit-promise");
var promisify = require("promisify-node");
var fse = promisify("fs-extra");
var zlib = require("zlib");
var cp = require("child_process");
var path = require("path");

var local = path.join.bind(path, __dirname);
var rooted = path.join.bind(path, __dirname, "..");

var check = require(local("checkPrepared")).checkVendor;
var pkg = require(rooted("package"));
var tar;
var request;

var NODE_VERSION = Number(process.version.match(/^v(\d+\.\d+)/)[1]);

module.exports = function retrieveExternalDependencies() {
  tar = require("tar");
  request = require("request");
  return Promise.all([
    getVendorLib("libgit2", "https://github.com/libgit2/libgit2/tarball/" + pkg.libgit2.sha),
    getVendorLib("libssh2", pkg.libssh2.url),
    getVendorLib("http_parser", pkg.http_parser.url)
  ])
};


function getVendorLib(name, url) {
  var vendorPath = "vendor/" + name + "/";
  var vendorPackage = pkg[name];
  if (NODE_VERSION === 0.1 && vendorPackage["0.10"]) {
    vendorPackage = vendorPackage["0.10"];
  }

  var version = vendorPackage.sha || vendorPackage.version;

  console.info("[nodegit] Detecting " + vendorPath + ".");
  return check(name)
    .then(function(exists) {
      if (exists) {
        console.info("[nodegit] " + vendorPath + " already exists.");
        return Promise.resolve();
      }
      else {
        return check(name, true)
          .then(function(exists) {
            if (exists) {
              console.info("[nodegit] Removing outdated " + vendorPath + ".");
              return fse.remove(rooted(vendorPath));
            }
            else {
              console.info("[nodegit] " + vendorPath + " not found.");
              return Promise.resolve();
            }
          })
          .then(function() {
            return new Promise(function (resolve, reject) {
              console.info("[nodegit] Retrieving " + vendorPath + ".");

              var extract = tar.Extract({
                path: rooted("vendor/" + name + "/"),
                strip: true
              });

              request.get(url)
                .pipe(zlib.createUnzip())
                .pipe(extract)
                .on("error", reject)
                .on("end", resolve);
            });

          })
          .then(function() {
            return fse.writeFile(rooted(vendorPath + version), "");
          })
          .then(function() {
            if ((name == "libssh2") && (process.platform !== "win32")) {
              return new Promise(function(resolve, reject) {
                console.info("[nodegit] Configuring libssh2.");
                cp.exec(rooted(vendorPath) + "configure", {cwd: rooted(vendorPath)}, function(err, stdout, stderr) {
                  if (err) {
                    console.error(err);
                    console.error(stderr);
                    reject(err, stderr);
                  }
                  else {
                    resolve(stdout);
                  }
                })
              });
            }
            else {
              return Promise.resolve();
            }
          })
          .then(function() {
            console.info("[nodegit] Successfully updated " + vendorPath + ".");
          });
      }
    });

}

// Called on the command line
if (require.main === module) {
  module.exports();
}<commit_msg>debug info<commit_after>var Promise = require("nodegit-promise");
var promisify = require("promisify-node");
var fse = promisify("fs-extra");
var zlib = require("zlib");
var cp = require("child_process");
var path = require("path");

var local = path.join.bind(path, __dirname);
var rooted = path.join.bind(path, __dirname, "..");

var check = require(local("checkPrepared")).checkVendor;
var pkg = require(rooted("package"));
var tar;
var request;

var NODE_VERSION = Number(process.version.match(/^v(\d+\.\d+)/)[1]);

module.exports = function retrieveExternalDependencies() {
  tar = require("tar");
  request = require("request");
  return Promise.all([
    getVendorLib("libgit2", "https://github.com/libgit2/libgit2/tarball/" + pkg.libgit2.sha),
    getVendorLib("libssh2", pkg.libssh2.url),
    getVendorLib("http_parser", pkg.http_parser.url)
  ])
};


function getVendorLib(name, url) {
  var vendorPath = "vendor/" + name + "/";
  var vendorPackage = pkg[name];
  if (NODE_VERSION === 0.1 && vendorPackage["0.10"]) {
    vendorPackage = vendorPackage["0.10"];
  }

  var version = vendorPackage.sha || vendorPackage.version;

  console.info("[nodegit] Detecting " + vendorPath + version + ".");
  return check(name)
    .then(function(exists) {
      if (exists) {
        console.info("[nodegit] " + vendorPath + version + " already exists.");
        return Promise.resolve();
      }
      else {
        return check(name, true)
          .then(function(exists) {
            if (exists) {
              console.info("[nodegit] Removing outdated " + vendorPath + ".");
              return fse.remove(rooted(vendorPath));
            }
            else {
              console.info("[nodegit] " + vendorPath + " not found.");
              return Promise.resolve();
            }
          })
          .then(function() {
            return new Promise(function (resolve, reject) {
              console.info("[nodegit] Retrieving " + vendorPath + ".");

              var extract = tar.Extract({
                path: rooted("vendor/" + name + "/"),
                strip: true
              });

              request.get(url)
                .pipe(zlib.createUnzip())
                .pipe(extract)
                .on("error", reject)
                .on("end", resolve);
            });

          })
          .then(function() {
            return fse.writeFile(rooted(vendorPath + version), "");
          })
          .then(function() {
            if ((name == "libssh2") && (process.platform !== "win32")) {
              return new Promise(function(resolve, reject) {
                console.info("[nodegit] Configuring libssh2.");
                cp.exec(rooted(vendorPath) + "configure", {cwd: rooted(vendorPath)}, function(err, stdout, stderr) {
                  if (err) {
                    console.error(err);
                    console.error(stderr);
                    reject(err, stderr);
                  }
                  else {
                    resolve(stdout);
                  }
                })
              });
            }
            else {
              return Promise.resolve();
            }
          })
          .then(function() {
            console.info("[nodegit] Successfully updated " + vendorPath + version + ".");
          });
      }
    });

}

// Called on the command line
if (require.main === module) {
  module.exports();
}
{'fraction_non_alphanumeric': 0.15424354243542435, 'fraction_numerical': 0.005608856088560886, 'mean_word_length': 1.880952380952381, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7143153', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1882, 'n_words': 368}