<commit_before>"use strict";

var Promise   = require('bluebird')
  , url       = require('url')
  , path      = require('path')
  , fs        = require('fs-extra')
  , _         = require('underscore')
  , Git       = require('nodegit')
  , Datastore = require('nedb')
  ;

var db = new Datastore({ filename: '.db/repository.db', autoload: true });

class Repository {
  constructor(id) {
    this.id = id;

    this.remote = id;

    var url_parsed = url.parse(this.id);

    this.path = path.join('.db', url_parsed.hostname, url_parsed.pathname);

    var path_parts = url_parsed.pathname.split(path.sep);
    
    this.username = path_parts[1];
    this.name = path.basename(path_parts[2], '.git');
  }

  cloneAt() {
    return Promise
            .promisify(fs.stat)(path.join(this.path, '.git/HEAD'))
            .then((stat) => {
              return stat.mtime.getTime();
            });
  }

  fetchAt() {
    return Promise
            .promisify(fs.stat)(path.join(this.path, '.git/FETCH_HEAD'))
            .then((stat) => {
              return stat.mtime.getTime();
            });
  }

  updatedAt() {
    return this.fetchAt()
            .catch(() => this.cloneAt() )
            .catch(() => undefined )
            ;
  }

  getMetadata() {
    return Promise.promisify(db.findOne, { context : db })({ remote : this.remote });
  }

  updateMetadata(patch) {
    var query   = { remote : this.remote }
      , data    = _.defaults({ remote : this.remote }, patch)
      , opts    = { upsert: true }
      ;

    return Promise.promisify(db.update, { context : db })(query, data, opts);
  }

  acl(options) {
    return this
            .getMetadata()
            .then((m) => {
              if (!_.isObject(m)) {
                throw new Error('not found');
              }

              var ret = Promise.resolve(m);

              if (m.private && options.token !== m.token) {
                // it is possible that the token was update / try a fetch and accept new token on success
                ret.tap(() => {
                  this.pull(options);
                });
              }

              return ret;
            });
  }

  onLocal() {
    return Promise
              .promisify(fs.access)(path.join(this.path, '.git/HEAD'), fs.R_OK | fs.W_OK)
              .then(() => {
                return true;
              })
              .catch(() => {
                return false;
              });
  }

  clone(options) {
    if (!this.c) {
      this.c = Promise
                .promisify(fs.ensureDir)(path.dirname(this.path))
                .then(() => {
                  var opt = { fetchOpts : { callbacks : { certificateCheck  : () => 1 } } };

                  return Promise
                          .resolve(Git.Clone(this.remote, this.path, opt))
                          .tap(() => {
                            return this.updateMetadata({ private : false });
                          })
                          .catch({ message : 'authentication required but no callback set' }, (err) => {
                            if (options.token) {
                              opt.fetchOpts.callbacks.credentials = () => Git.Cred.userpassPlaintextNew(options.token, 'x-oauth-basic');
                              
                              return Promise
                                      .resolve(Git.Clone(this.remote, this.path, opt))
                                      .tap(() => {
                                        return this.updateMetadata({ private : true, token : options.token });
                                      });
                            } else {
                              throw err;
                            }
                          });
                })
                .catch((err) => {
                  delete Repository.store[this.id];
                  // mask error
                  console.error(this.remote, err);
                  throw new Error('not found');
                })
                .finally(() => {
                  delete this.c;
                });
    }

    return this.c;
  }

  // todo [akamel] problem when clone with token A fails / chained clone with token B will also fail and must retry
  pull(options) {
    var ret = this.p || this.c;

    if (!ret) {
      ret = this
              .onLocal()
              .then((local) => {
                if (!local) {
                  return this.clone({ token : options.token });
                }

                this.p = Git.Repository
                          .open(this.path)
                          .then((repo) => {
                            var opt = { callbacks : { certificateCheck  : () => 1 } };
                            
                            if (options.token) {
                              opt.callbacks.credentials = () => Git.Cred.userpassPlaintextNew(options.token, 'x-oauth-basic');
                            }

                            return Promise
                                    .resolve(repo.fetch('origin', opt))
                                    .then(() => {
                                      return repo.mergeBranches('master', 'origin/master');
                                    })
                                    .tap(() => {
                                      return this
                                              .getMetadata()
                                              .then((m) => {
                                                // update token in case it changed
                                                if (m && m.token) {
                                                  return this.updateMetadata({ token : options.token });
                                                }
                                              });
                                    });
                          })
                          .catch((err) => {
                            throw new Error('not found', this.id);
                          })
                          .finally(() => {
                            delete this.p;
                          });

                return this.p;
              });
    }

    return ret;
  }

  getMasterCommit() {
    return Git.Repository
            .open(this.path)
            .then((repo) => {
              return repo.getMasterCommit()
            });
  }

  entry(name) {
    return this
            .getMasterCommit()<commit_msg>reset repo before a fetch...<commit_after>"use strict";

var Promise   = require('bluebird')
  , url       = require('url')
  , path      = require('path')
  , fs        = require('fs-extra')
  , _         = require('underscore')
  , Git       = require('nodegit')
  , Reset     = Git.Reset
  , Datastore = require('nedb')
  ;

var db = new Datastore({ filename: '.db/repository.db', autoload: true });

class Repository {
  constructor(id) {
    this.id = id;

    this.remote = id;

    var url_parsed = url.parse(this.id);

    this.path = path.join('.db', url_parsed.hostname, url_parsed.pathname);

    var path_parts = url_parsed.pathname.split(path.sep);
    
    this.username = path_parts[1];
    this.name = path.basename(path_parts[2], '.git');
  }

  cloneAt() {
    return Promise
            .promisify(fs.stat)(path.join(this.path, '.git/HEAD'))
            .then((stat) => {
              return stat.mtime.getTime();
            });
  }

  fetchAt() {
    return Promise
            .promisify(fs.stat)(path.join(this.path, '.git/FETCH_HEAD'))
            .then((stat) => {
              return stat.mtime.getTime();
            });
  }

  updatedAt() {
    return this.fetchAt()
            .catch(() => this.cloneAt() )
            .catch(() => undefined )
            ;
  }

  getMetadata() {
    return Promise.promisify(db.findOne, { context : db })({ remote : this.remote });
  }

  updateMetadata(patch) {
    var query   = { remote : this.remote }
      , data    = _.defaults({ remote : this.remote }, patch)
      , opts    = { upsert: true }
      ;

    return Promise.promisify(db.update, { context : db })(query, data, opts);
  }

  acl(options) {
    return this
            .getMetadata()
            .then((m) => {
              if (!_.isObject(m)) {
                throw new Error('not found');
              }

              var ret = Promise.resolve(m);

              if (m.private && options.token !== m.token) {
                // it is possible that the token was update / try a fetch and accept new token on success
                ret.tap(() => {
                  this.pull(options);
                });
              }

              return ret;
            });
  }

  onLocal() {
    return Promise
              .promisify(fs.access)(path.join(this.path, '.git/HEAD'), fs.R_OK | fs.W_OK)
              .then(() => {
                return true;
              })
              .catch(() => {
                return false;
              });
  }

  clone(options) {
    if (!this.c) {
      this.c = Promise
                .promisify(fs.ensureDir)(path.dirname(this.path))
                .then(() => {
                  var opt = { fetchOpts : { callbacks : { certificateCheck  : () => 1 } } };

                  return Promise
                          .resolve(Git.Clone(this.remote, this.path, opt))
                          .tap(() => {
                            return this.updateMetadata({ private : false });
                          })
                          .catch({ message : 'authentication required but no callback set' }, (err) => {
                            if (options.token) {
                              opt.fetchOpts.callbacks.credentials = () => Git.Cred.userpassPlaintextNew(options.token, 'x-oauth-basic');
                              
                              return Promise
                                      .resolve(Git.Clone(this.remote, this.path, opt))
                                      .tap(() => {
                                        return this.updateMetadata({ private : true, token : options.token });
                                      });
                            } else {
                              throw err;
                            }
                          });
                })
                .catch((err) => {
                  delete Repository.store[this.id];
                  // mask error
                  console.error(this.remote, err);
                  throw new Error('not found');
                })
                .finally(() => {
                  delete this.c;
                });
    }

    return this.c;
  }

  // todo [akamel] problem when clone with token A fails / chained clone with token B will also fail and must retry
  pull(options) {
    var ret = this.p || this.c;

    if (!ret) {
      ret = this
              .onLocal()
              .then((local) => {
                if (!local) {
                  return this.clone({ token : options.token });
                }

                this.p = Git.Repository
                          .open(this.path)
                          .then((repo) => {
                            return Promise
                                    .resolve(repo.getBranchCommit('origin/master'))
                                    .then((originHeadCommit) => {
                                      return Reset.reset(repo, originHeadCommit, Reset.TYPE.HARD);
                                    })
                                    .then(() => {
                                      var opt = { callbacks : { certificateCheck  : () => 1 } };
                                      
                                      if (options.token) {
                                        opt.callbacks.credentials = () => Git.Cred.userpassPlaintextNew(options.token, 'x-oauth-basic');
                                      }

                                      return repo.fetch('origin', opt);
                                    })
                                    .then(() => {
                                      return repo.mergeBranches('master', 'origin/master');
                                    })
                                    .tap(() => {
                                      return this
                                              .getMetadata()
                                              .then((m) => {
                                                // todo [akamel] this doesn't deal with repos that change from public to private?
                                                // update token in case it changed
                                                if (m && m.token) {
                                                  return this.updateMetadata({ token : options.token });
                                                }
                                              });
                                    });
                          })
                          .catch((err) => {
                            console.error(err);
                            // todo [akamel] this masks many errors
                            throw new Error('not found', this.id);
                          })
                          .finally(() => {
                            delete this.p;
                          });

                return this.p;
              });
    }

    return ret;
  }

  getMasterCommit() {
    return Git.Repository
            .open(this.path)
            .then((repo) => {
              return repo.getMasterCommit()
            });
  }

  entry(name) {
    return this
            .getMasterCommit()
{'fraction_non_alphanumeric': 0.11943914256350022, 'fraction_numerical': 0.0005872852738217589, 'mean_word_length': 0.9222520107238605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4784582', 'n_tokens_mistral': 3356, 'n_tokens_neox': 3223, 'n_words': 667}