<commit_before>// Bootstrap the platform to run on node.js
require('../lib/platform.js')(require('./node'));

// Load the libraries
var fsDb = require('../lib/fs-db.js');
var wrap = require('../lib/repo.js');
var each = require('../helpers/each.js');
var autoProto = require('../protocols/auto.js');
var urlParse = require('url').parse;

var url = process.argv[2] || "git://github.com/creationix/conquest.git";
var opts = urlParse(url);
if (!opts.protocol) {
  opts = urlParse("ssh://" + url);
}
var path = opts.pathname.match(/[^\/]*$/)[0];

var connection = autoProto(opts);
var repo = wrap(fsDb(path, true));

connection.discover(function (err, result) {
  if (err) throw err;
  var refs = result.refs;
  var wants = [];
  each(refs, function (name, hash) {
    if (name === "HEAD" || name.indexOf('^') > 0) return;
    wants.push("want " + hash);
  });

  connection.negotiate(wants, {
    serverCaps: result.caps,
    includeTag: true,
    // onProgress: onProgress,
    onError: function (data) {
      process.stderr.write(data);
    }
  }, function (err, packStream) {
    if (err) throw err;
    repo.init(function (err) {
      if (err) throw err;
      repo.importRefs(refs, function (err) {
        if (err) throw err;
        repo.unpack(packStream, {
          // onProgress: onProgress,
        }, function (err) {
          if (err) throw err;
          console.log("DONE");
        });
      });
    });
  });
});

function onProgress(data) {
  process.stdout.write(data);
}
<commit_msg>Fix clone example to use new api<commit_after>// Bootstrap the platform to run on node.js
require('../lib/platform.js')(require('./node'));

// Load the libraries
var fsDb = require('../lib/fs-db.js');
var wrap = require('../lib/repo.js');
var each = require('../helpers/each.js');
var autoProto = require('../protocols/auto.js');
var urlParse = require('url').parse;

var url = process.argv[2] || "git://github.com/creationix/conquest.git";
var opts = urlParse(url);
if (!opts.protocol) {
  opts = urlParse("ssh://" + url);
}
var path = opts.pathname.match(/[^\/]*$/)[0];

var connection = autoProto(opts);
var repo = wrap(fsDb(path, true));

connection.discover(function (err, result) {
  if (err) throw err;
  var refs = result.refs;
  var wants = [];
  each(refs, function (name, hash) {
    if (name === "HEAD" || name.indexOf('^') > 0) return;
    wants.push("want " + hash);
  });

  connection.fetch(wants, {
    serverCaps: result.caps,
    includeTag: true,
    // onProgress: onProgress,
    onError: function (data) {
      process.stderr.write(data);
    }
  }, function (err, packStream) {
    if (err) throw err;
    repo.init(function (err) {
      if (err) throw err;
      repo.importRefs(refs, function (err) {
        if (err) throw err;
        repo.unpack(packStream, {
          // onProgress: onProgress,
        }, function (err) {
          if (err) throw err;
          console.log("DONE");
        });
      });
    });
  });
});

function onProgress(data) {
  process.stdout.write(data);
}

{'fraction_non_alphanumeric': 0.18921601058551107, 'fraction_numerical': 0.0019847833278200462, 'mean_word_length': 2.9120310478654594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4964553', 'n_tokens_mistral': 1027, 'n_tokens_neox': 1011, 'n_words': 240}