<commit_before>var fs = require('fs');
var path = require('path');
var urlParse = require('url').parse;
var format = require('util').format;
var tcp = require('js-git/net/tcp-node');
var tcpTransport = require('js-git/net/transport-tcp')(tcp);
var request = require('../lib/js-git-node-request');
var httpTransport = require('js-git/net/transport-http')(request);
var fetchPackProtocol = require('js-git/net/git-fetch-pack');
var modes = require('js-git/lib/modes');
var archiver = require('archiver');
var del = require('del');
var mkdirp = require('mkdirp');
var ncp = require('ncp').ncp;
var config = require('../lib/config').get();
var utils = require('../lib/utils');

module.exports = {
  remote: function(repoUrl) {
    if (!repoUrl.startsWith('git:') && !repoUrl.startsWith('https:')) {
      var gitDir = path.join(repoUrl, '.git');
      if (fs.existsSync(gitDir)) {
        repoUrl = gitDir;
      }
      return {
        url: {
          href: repoUrl,
          pathname: repoUrl,
          protocol: 'file:'
        },
        transport: 'fs'
      };
    }
    
    repoUrl = utils.fixUrl(repoUrl);
    var parsedUrl = urlParse(repoUrl);
    var transport, transportName;
    
    if (parsedUrl.protocol == 'git:') {
      transport = tcpTransport(
        parsedUrl.pathname,
        parsedUrl.hostname,
        parsedUrl.port
      );
      transportName = 'tcp';
    } else {
      transport = httpTransport(repoUrl);
      transportName = 'http';
    }

    var api = fetchPackProtocol(transport);
    
    // add helpers
    api.url = parsedUrl;
    api.transport = transportName;
    api.done = function() {
      this.put({ done: true });
      this.put();
    };
    
    return api;
  },
  repo: function(rootPath, mirror) {
      var repo = {};
      var fsDb = require('../lib/js-git-fs-db');
      require('js-git/mixins/fs-db')(repo, fsDb);
      require('js-git/mixins/read-combiner')(repo);
      require('js-git/mixins/pack-ops')(repo);
      require('js-git/mixins/walkers')(repo);
      require('js-git/mixins/formats')(repo);
      
      if (!rootPath) {
        rootPath = process.cwd();
        var gitDir = path.join(rootPath, '.git');
        if (fs.existsSync(gitDir)) {
          rootPath = gitDir;
        }
      } else if (!mirror) {
        rootPath = path.join(rootPath, '.git');
      }
      
      repo.rootPath = rootPath;
      
      return repo;
  },
  ls: function(remote, callback, keepalive) {
    remote.take(function(err, refs) {
      if (remote.transport == 'tcp' && !keepalive) {
        remote.done();
      }
      callback(err, refs);
    });
  },
  clone: function(remote, targetPath, options, callback) {
    var self = this;
    targetPath = targetPath || path.basename(remote.url.pathname, '.git');
    
    if (options.onProgress) {
      var progress;
      if (options.mirror) {
        progress = "Cloning '%s' into bare repository '%s'...";
      } else {
        progress = "Cloning '%s' into '%s'...";
      }
      options.onProgress(format(progress, remote.url.href, targetPath));
    }
    
    var repo = this.repo(targetPath, options.mirror);
    
    if (remote.transport == 'fs') {
      return mkdirp(repo.rootPath, function(err) {
        if (err) return callback(err);
        ncp(remote.url.href, repo.rootPath, { limit: 16 }, function(err) {
          if (err) return callback(err);
          self.checkout(repo, /*head*/null, options, callback);
        });
      });
    }
    
    var branch = options.branch || 'refs/heads/master';
    if (!branch.startsWith('refs/') && branch[0] != '/') {
      branch = '/' + branch;
    }
    
    var ref, hash;
    this.ls(remote, select, /*keepalive*/true);

    function select(err, refs) {
      if (err) return callback(err);
    
      ref = Object.keys(refs).filter(function(ref) {
        return ref.endsWith(branch); 
      })[0];
      hash = refs[ref];
      remote.put({ want: hash });
      
      if (options.depth) {
        remote.put({ deepen: parseInt(options.depth, 10) });
      }
      
      remote.put(null);
      remote.done();
      remote.take(unpack);
    }
    
    function unpack(err, channels) {
      if (err) return callback(err);
      
      var packOpts = {
        onError: callback,
        onProgress: function(progress) {<commit_msg>Clone: get all heads and tags from remote if branch is not specified<commit_after>var fs = require('fs');
var path = require('path');
var urlParse = require('url').parse;
var format = require('util').format;
var tcp = require('js-git/net/tcp-node');
var tcpTransport = require('js-git/net/transport-tcp')(tcp);
var request = require('../lib/js-git-node-request');
var httpTransport = require('js-git/net/transport-http')(request);
var fetchPackProtocol = require('js-git/net/git-fetch-pack');
var modes = require('js-git/lib/modes');
var archiver = require('archiver');
var del = require('del');
var mkdirp = require('mkdirp');
var ncp = require('ncp').ncp;
var config = require('../lib/config').get();
var utils = require('../lib/utils');

module.exports = {
  remote: function(repoUrl) {
    if (!repoUrl.startsWith('git:') && !repoUrl.startsWith('https:')) {
      var gitDir = path.join(repoUrl, '.git');
      if (fs.existsSync(gitDir)) {
        repoUrl = gitDir;
      }
      return {
        url: {
          href: repoUrl,
          pathname: repoUrl,
          protocol: 'file:'
        },
        transport: 'fs'
      };
    }
    
    repoUrl = utils.fixUrl(repoUrl);
    var parsedUrl = urlParse(repoUrl);
    var transport, transportName;
    
    if (parsedUrl.protocol == 'git:') {
      transport = tcpTransport(
        parsedUrl.pathname,
        parsedUrl.hostname,
        parsedUrl.port
      );
      transportName = 'tcp';
    } else {
      transport = httpTransport(repoUrl);
      transportName = 'http';
    }

    var api = fetchPackProtocol(transport);
    
    // add helpers
    api.url = parsedUrl;
    api.transport = transportName;
    api.done = function() {
      this.put({ done: true });
      this.put();
    };
    
    return api;
  },
  repo: function(rootPath, mirror) {
      var repo = {};
      var fsDb = require('../lib/js-git-fs-db');
      require('js-git/mixins/fs-db')(repo, fsDb);
      require('js-git/mixins/read-combiner')(repo);
      require('js-git/mixins/pack-ops')(repo);
      require('js-git/mixins/walkers')(repo);
      require('js-git/mixins/formats')(repo);
      
      if (!rootPath) {
        rootPath = process.cwd();
        var gitDir = path.join(rootPath, '.git');
        if (fs.existsSync(gitDir)) {
          rootPath = gitDir;
        }
      } else if (!mirror) {
        rootPath = path.join(rootPath, '.git');
      }
      
      repo.rootPath = rootPath;
      
      return repo;
  },
  ls: function(remote, callback, keepalive) {
    remote.take(function(err, refs) {
      if (remote.transport == 'tcp' && !keepalive) {
        remote.done();
      }
      callback(err, refs);
    });
  },
  clone: function(remote, targetPath, options, callback) {
    var self = this;
    targetPath = targetPath || path.basename(remote.url.pathname, '.git');
    
    if (options.onProgress) {
      var progress;
      if (options.mirror) {
        progress = "Cloning '%s' into bare repository '%s'...";
      } else {
        progress = "Cloning '%s' into '%s'...";
      }
      options.onProgress(format(progress, remote.url.href, targetPath));
    }
    
    var repo = this.repo(targetPath, options.mirror);
    
    if (remote.transport == 'fs') {
      return mkdirp(repo.rootPath, function(err) {
        if (err) return callback(err);
        ncp(remote.url.href, repo.rootPath, { limit: 16 }, function(err) {
          if (err) return callback(err);
          self.checkout(repo, /*head*/null, options, callback);
        });
      });
    }
    
    var branch = options.branch;
    if (branch && !branch.startsWith('refs/') && branch[0] != '/') {
      branch = '/' + branch;
    }
    
    var ref, hash;
    this.ls(remote, select, /*keepalive*/true);

    function select(err, refs) {
      if (err) return callback(err);
    
      var selectedRefs = Object.keys(refs).filter(function(ref) {
        if (branch) {
          return ref.endsWith(branch); 
        } else {
          return ref.startsWith('refs/heads/') || ref.startsWith('refs/tags/');
        }
      });
      
      if (branch) {
        ref = selectedRefs[0];
      } else {
        ref = 'refs/heads/master';
      }
      
      hash = refs[ref];
      
      selectedRefs.forEach(function(ref) {
        remote.put({ want: refs[ref] });
      });
      
      if (options.depth) {
        remote.put({ deepen: parseInt(options.depth, 10) });
      }
      
      remote.put(null);
      remote.done();
      remote.take(unpack);
    }
    
    function unpack(err, channels) {
      if (err) return callback(err);
      
      var packOpts = {
        onError: callback,
        onProgress: function(progress) {
{'fraction_non_alphanumeric': 0.16278810822848236, 'fraction_numerical': 0.0013361541031065582, 'mean_word_length': 2.2792990142387732, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4394057', 'n_tokens_mistral': 2885, 'n_tokens_neox': 2881, 'n_words': 578}