<commit_before>var express     = require('express')
  , Promise     = require('bluebird')
  , bodyParser  = require('body-parser')
  , winston     = require('winston')
  , config      = require('config-url')
  , _           = require('underscore')
  , man         = require('taskmill-core-man')
  , rp          = require('request-promise')
  , babel       = require('babel-core')
  , Repository  = require('./git/repository')
  , urljoin     = require('url-join')
  ;

var app = express();

app.use(bodyParser.json());

app.post('/blob', function(req, res, next){
  var remote          = req.body.remote   //|| 'https://github.com/a7medkamel/taskmill-core-agent.git'
    , branch          = req.body.branch   || 'master' // todo [akamel] not used
    , filename        = req.body.filename //|| 'lib/core/worker.js'
    , token           = req.body.token
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('blob', 'at', repo.id, at);
                if (!at || Date.now - at > config.get('codedb.ttl')) {
                  return repo.pull({ token : token }).then(() => Date.now);
                }

                return at;
              })
              .then((at) => {
                return [ at, repo.acl({ token : token }) ];
              })
              .spread((at, repository) => {
                // todo [akamel] this can leak repo existance, mask with 'file not found'
                return repo
                        .stat(filename)
                        .then((stat) => {
                          if (stat.rawsize > 1024 /* 1kb */ * 10) {
                            throw new Error('file is larger than 10kb limit');
                          }

                          if (stat.isBinary) {
                            throw new Error('file is binary');
                          }
                          return [ at, stat, repository ];
                        });
              })
              .spread((at, stat, repository) => {
                return repo
                        .cat(filename)
                        .then((data) => {
                          res.send({
                              stat        : stat
                            , repository  : _.omit(repository, '_id', 'token')
                            // todo [akamel] this doesn't have branch/commit
                            , uid         : urljoin(remote + '+', filename)
                            // todo [akamel] add branch and path to ls as well...
                            , branch      : branch
                            , path        : filename[0] === '/'? filename : '/' + filename
                            , updatedAt   : at
                            , content     : data
                          });
                        });
              });
    })
    .catch(function(err){
      winston.error(err);
      res.status(400).send({ message : err.message });
    });
});

app.post('/pull', function(req, res, next){
  var remote          = req.body.remote
    , token           = req.body.token
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('pull', 'at', repo.id, at);
                if (at) {
                  return repo.pull({ token : token });
                }
              })
              .then(() => {
                res.send({ message : 'OK' });
              })
    })
    .catch(function(err){
      winston.error(err);
      res.status(400).send({ message : err.message });
    });
});

app.post('/ls', function(req, res, next){
  var remote          = req.body.remote
    , token           = req.body.token
    , populate        = req.body.populate
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('ls', 'at', repo.id, at);
                if (!at) {
                  return repo.pull({ token : token }).then(() => at);
                }

                return at;
              })
              .tap(() => {
                return repo.acl({ token : token });
              })
              .then((at) => {
                return repo
                        .ls()
                        .then((result) => {
                          return {
                              updatedAt : at
                            , data      : result.data
                          }
                        });
              })
              .then((result) => {
                if (_.isEmpty(populate)) {
                  return result;
                }

                let limited = _.take(result.data, config.get('codedb.populate.limit'));
                return Promise
                        .map(limited, (item) => {
                          if (populate.repository) {
                            item.repository = repo.name;
                          }

                          if (populate.username) {
                            item.username = repo.username;
                          }

                          return repo
                                  .stat(item.path)
                                  .then((stat) => {
                                    if (stat.rawsize > 1024 /* 1kb */ * 10) {
                                      item.rawsize = stat.rawsize;
                                      return item;
                                    }

                                    if (stat.isBinary) {
                                      item.binary = true;
                                      return item;
                                    }

                                    return repo
                                            .cat(item.path)
                                            .then((text) => {
                                              if (populate.blob) {
                                                item.blob = text;
                                              }

                                              let es6 = undefined;
                                              if (populate.ast) {
                                                es6 = es6 || babel.transform(text); // => { code, map, ast }
                                                item.ast = es6.ast;
                                              }

                                              if (populate.manual) {
                                                es6 = es6 || babel.transform(text); // => { code, map, ast }
                                                let manual = man.get(es6);
                                                if (!_.isEmpty(manual)) {
                                                  item.manual = manual
                                                }
                                              }

                                              if (populate.es5) {
                                                es6 = es6 || babel.transform(text); // => { code, map, ast }
                                                item.es5 = es6.code;
                                              }

                                              return item;
                                            })
                                            .catch((err) => {
                                              // supress errors in populate
                                              return item;
                                            });
                                  });<commit_msg>add populate manual to blob<commit_after>var express     = require('express')
  , Promise     = require('bluebird')
  , bodyParser  = require('body-parser')
  , winston     = require('winston')
  , config      = require('config-url')
  , _           = require('underscore')
  , man         = require('taskmill-core-man')
  , rp          = require('request-promise')
  , babel       = require('babel-core')
  , Repository  = require('./git/repository')
  , urljoin     = require('url-join')
  ;

var app = express();

app.use(bodyParser.json());

function blob_metadata(text, fields) {
  fields = fields || {};

  let ret = {};

  if (fields.ast || fields.manual || fields.es5) {
    try {
      let es6 = babel.transform(text); // => { code, map, ast }

      if (fields.ast) {
        ret.ast = es6.ast;
      }

      if (fields.manual) {
        let manual = man.get(es6);
        if (!_.isEmpty(manual)) {
          ret.manual = manual
        }
      }

      if (fields.es5) {
        ret.es5 = es6.code;
      }
    } catch(err) {}
  }

  return ret;
}

app.post('/blob', function(req, res, next){
  var remote          = req.body.remote   //|| 'https://github.com/a7medkamel/taskmill-core-agent.git'
    , branch          = req.body.branch   || 'master' // todo [akamel] not used
    , filename        = req.body.filename //|| 'lib/core/worker.js'
    , token           = req.body.token
    , populate        = req.body.populate || {}
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('blob', 'at', repo.id, at);
                if (!at || Date.now - at > config.get('codedb.ttl')) {
                  return repo.pull({ token : token }).then(() => Date.now);
                }

                return at;
              })
              .then((at) => {
                return [ at, repo.acl({ token : token }) ];
              })
              .spread((at, repository) => {
                // todo [akamel] this can leak repo existance, mask with 'file not found'
                return repo
                        .stat(filename)
                        .then((stat) => {
                          if (stat.rawsize > 1024 /* 1kb */ * 10) {
                            throw new Error('file is larger than 10kb limit');
                          }

                          if (stat.isBinary) {
                            throw new Error('file is binary');
                          }
                          return [ at, stat, repository ];
                        });
              })
              .spread((at, stat, repository) => {
                return repo
                        .cat(filename)
                        .then((text) => {
                          let item = {
                              stat        : stat
                            , repository  : _.omit(repository, '_id', 'token')
                            // todo [akamel] this doesn't have branch/commit
                            , uid         : urljoin(remote + '+', filename)
                            // todo [akamel] add branch and path to ls as well...
                            , branch      : branch
                            , path        : filename[0] === '/'? filename : '/' + filename
                            , updatedAt   : at
                            , content     : text
                          };

                          let meta = blob_metadata(text, populate);

                          res.send(_.extend(item, meta));
                        });
              });
    })
    .catch(function(err){
      winston.error(err);
      res.status(400).send({ message : err.message });
    });
});

app.post('/pull', function(req, res, next){
  var remote          = req.body.remote
    , token           = req.body.token
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('pull', 'at', repo.id, at);
                if (at) {
                  return repo.pull({ token : token });
                }
              })
              .then(() => {
                res.send({ message : 'OK' });
              })
    })
    .catch(function(err){
      winston.error(err);
      res.status(400).send({ message : err.message });
    });
});

app.post('/ls', function(req, res, next){
  var remote          = req.body.remote
    , token           = req.body.token
    , populate        = req.body.populate
    ;

  Repository
    .get(remote)
    .then((repo) => {
      return repo
              .updatedAt()
              .then((at) => {
                winston.verbose('ls', 'at', repo.id, at);
                if (!at) {
                  return repo.pull({ token : token }).then(() => at);
                }

                return at;
              })
              .tap(() => {
                return repo.acl({ token : token });
              })
              .then((at) => {
                return repo
                        .ls()
                        .then((result) => {
                          return {
                              updatedAt : at
                            , data      : result.data
                          }
                        });
              })
              .then((result) => {
                if (_.isEmpty(populate)) {
                  return result;
                }

                let limited = _.take(result.data, config.get('codedb.populate.limit'));
                return Promise
                        .map(limited, (item) => {
                          if (populate.repository) {
                            item.repository = repo.name;
                          }

                          if (populate.username) {
                            item.username = repo.username;
                          }

                          return repo
                                  .stat(item.path)
                                  .then((stat) => {
                                    if (stat.rawsize > 1024 /* 1kb */ * 10) {
                                      item.rawsize = stat.rawsize;
                                      return item;
                                    }

                                    if (stat.isBinary) {
                                      item.binary = true;
                                      return item;
                                    }

                                    return repo
                                            .cat(item.path)
                                            .then((text) => {
                                              if (populate.blob) {
                                                item.blob = text;
                                              }

                                              let meta = blob_metadata(text, populate);

                                              _.extend(item, meta);
                                              
                                              return item;
                                            })
                                            .catch((err) => {
                                              // supress errors in populate
                                              return item;
                                            });
                                  });
{'fraction_non_alphanumeric': 0.11484834467719553, 'fraction_numerical': 0.0044274102953809555, 'mean_word_length': 0.7403403863845446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 108, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2674987', 'n_tokens_mistral': 3540, 'n_tokens_neox': 3308, 'n_words': 705}