<commit_before>import GithubAPI from 'github-api';
import _ from 'lodash';
import Bluebird from 'bluebird';
import mongoose from 'mongoose';
import _debug from 'debug';

const ObjectId = mongoose.Types.ObjectId;
const debug = _debug('jekyllpro-cms:repository');
mongoose.Promise = Bluebird;
const RepoFileEntry = mongoose.model('RepoFileEntry');
const lodash = _; // TODO

import {
  TaskQueue,
  getCollectionFiles,
  getCollectionType,
  getLangFromConfigYaml
} from './utils';
import { RepoIndex, RepoAccessToken } from './database';
import { hookConfig } from './webhook';

const RUNNING_JOBS = {};

const cb = (error, result, request) => {
  console.log(error);
  console.log(result);
};

/**
 * middleware
 * attach github-api instance to `req` - req.githubRepo
 */
const requireGithubAPI = (req, res, next) => {
  if (req.githubRepo) {
    return next();
  }
  var repoOwner = req.get('X-REPO-OWNER');
  var repoName = req.get('X-REPO-NAME');
  if (!repoName || !repoOwner) {
    res.status(401).send({ message: 'repository undefined' });
  }
  var ak =
    req.get('X-TOKEN') || req.user.accessToken || req.user._json.accessToken;
  var github = new GithubAPI({
    token: ak
  });
  req.githubRepo = github.getRepo(repoOwner, repoName);
  next();
};

const getRepoDetails = (req, res, next) => {
  var repo = req.githubRepo;

  repo
    .getDetails()
    .then(data => {
      // console.log(data)
      var details = _.pick(data.data, [
        'name',
        'full_name',
        'description',
        'private',
        'url',
        'html_url',
        'default_branch',
        'created_at',
        'updated_at',
        'pushed_at',
        'permissions'
      ]);
      details.owner = _.pick(data.data.owner, ['login', 'avatar_url', 'type']);
      res.status(200).json(details);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.response.status).json(err.response.data);
    });
};

const getRepoContent = (req, res, next) => {
  var repo = req.githubRepo;
  var { branch, path, raw } = req.query;

  repo
    .getContents(branch, path, raw)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const writeRepoFile = (req, res, next) => {
  var repo = req.githubRepo;
  var newFile = req.body;
  if (!newFile.options) {
    newFile.options = { encode: true };
  }
  repo
    .writeFile(
      newFile.branch,
      newFile.path,
      newFile.content,
      newFile.message,
      newFile.options,
      cb
    )
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const deleteRepoFile = (req, res, next) => {
  var repo = req.githubRepo;
  var toBeDeleted = req.body;

  repo
    .deleteFile(toBeDeleted.branch, toBeDeleted.path, cb)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const listBranches = (req, res, next) => {
  var repo = req.githubRepo;
  var { branch } = req.query;

  var nextPromise = branch ? repo.getBranch(branch) : repo.listBranches();

  return nextPromise
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res
        .status(err.status || err.response.status || err.response.status)
        .json(err.response.data);
    });
};

const createBranches = (req, res, next) => {
  var repo = req.githubRepo;
  var formData = req.body;

  repo
    .createBranch(formData.oldBranch, formData.newBranch)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const getBranchSchema = (req, res, next) => {
  var repo = req.githubRepo;
  var { ref, path } = req.query;
  path = path || '_schema';

  repo
    .getContents(ref, path)
    .then(data => {
      let schemaFiles = data.data
        .filter(item => {
          // filter out folder files
          return item.type === 'file';
        })
        .map(item => {
          return repo.getContents(ref, item.path, true).then(data => {
            return { name: item.name, data: data.data };
          });
        });

      return Promise.all(schemaFiles).then(results => {
        res.status(200).json(results);
      });
    })
    .catch(err => {
      console.log(err);
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const getRepoBranchIndex = (req, res, next) => {
  var repo = req.githubRepo;
  var branch = req.query.branch || 'master';
  var refreshIndex = req.query.refresh === true || req.query.refresh === 'true';
  var repoFullname = req.get('X-REPO-OWNER') + '/' + req.get('X-REPO-NAME');

  // update access token in db, which can be used to run the webhook service
  RepoAccessToken.findOneAndUpdate(
    {
      repository: repoFullname
    },
    {
      repository: repoFullname,
      accessToken: req.user.accessToken,
      updatedBy: req.user.login,
      updated: Date()
    },
    {
      upsert: true
    },
    err => {
      if (err) console.log(err);
    }
  );

  if (refreshIndex) {
    // set flag to clean old db record later if a new index cannot be built
    req.purgeDb = true;
    return next();
  }

  RepoIndex.findByRepoInfo(repoFullname, branch, (err, record) => {
    if (err) {
      console.log(err);
      return next();
    }
    if (!record) {
      return next();
    }
    debug(
      'get database record:',
      record.repository,
      record.branch,
      record.updated,
      record.tipCommitSha
    );
    // some repo branches might have legacy index data built when it didn't have schemas.
    // only return record if it has schemas.

    // collections now should be an array though unpopulated
    if (Array.isArray(record.collections) && record.tipCommitSha) {
      record
        .populate({
          path: 'collections',
          options: {
            sort: {
              lastUpdatedAt: 'desc'
            }
          }
        })
        .execPopulate()
        .then(record => {
          // @see mongoose doc
          // http://mongoosejs.com/docs/api.html#document_Document-execPopulate
          debug('record id is %s', record._id);
          let data = {
            updated: record.updated,
            collections: record.collections,
            schemas: JSON.parse(record.schemas),
            config: JSON.parse(record.config)
          };
          return res.status(200).json(data);
        });
    } else {
      debug('record invalid');
      // set flag to clean old db record later if a new index cannot be built
      req.purgeDb = true;
      // next middleware should be refreshIndexAndSave()
      return next();
    }
  });
};

function getRepoBranchUpdatedCollections(req, res) {
  const repo = req.githubRepo;
  const branch = req.query.branch || 'master';
  const repoFullname = req.get('X-REPO-OWNER') + '/' + req.get('X-REPO-NAME');
  // TODO
  // currently refreshIndexIncremental will always resolve
  // but we need to figure
  return refreshIndexIncremental({
    github: repo,
    repoFullname,
    branch
  })
    .then(updatedContent => res.status(200).json(updatedContent))
    .catch(
      err =>
        console.log(err) ||
        res.status(err.status || err.response.status).json(err.response.data)
    );
}

/**
 * @return {Promise<collections>} - {collections: { modified: Array, removed: Array }}
 */
function refreshIndexIncremental({ github, repoFullname, branch }) {
  debug(`refreshIndexIncremental ${repoFullname}:${branch}`);
  let repoIndex = null;
  let schemaArray = null;
  let schemaUpdated = false;
  let tipCommitSha = null;
  let newSchemas = [];
  const updatedContent = {
    collections: {
      modified: [], // consider added as modified
      removed: []
    }
  };

  // throttle
  const jobKey = `refreshIndexIncremental:${repoFullname}:${branch}`;
  if (RUNNING_JOBS[jobKey]) {
    debug('another refreshIndexIncremental task is running, returning');
    return Promise.resolve(updatedContent);
  }
  RUNNING_JOBS[jobKey] = true;
  const dispose = () => {
    delete RUNNING_JOBS[jobKey];
  };

  // get last commit
  return Promise.all([
    RepoIndex.findOne({ repository: repoFullname, branch }),
    github.getRef(`heads/${branch}`)
  ])
    .then(([_repoIndex, refData]) => {
      if (!_repoIndex.tipCommitSha) {
        debug(
          'Error: no tipCommitSha on %s repoIndex %s',
          repoFullname,
          _repoIndex._id
        );
        throw 'break';
      }

      tipCommitSha = lodash.get(refData, 'data.object.sha');
      // can not get the commit sha
      if (typeof tipCommitSha !== 'string') throw 'break';
      // the commit sha is same as the one in database
      if (tipCommitSha === _repoIndex.tipCommitSha) throw 'break';

      debug(
        `tip commit: db is ${_repoIndex.tipCommitSha} remote is ${tipCommitSha}`
      );
      repoIndex = _repoIndex;
      schemaArray = JSON.parse(repoIndex.schemas);

      // doc https://developer.github.com/v3/repos/commits/#compare-two-commits
      return github.compareBranches(repoIndex.tipCommitSha, branch);
    })
    .then(data => {
      debug(
        '%d commits between %s and %s',
        data.data.total_commits,
        repoIndex.tipCommitSha,
        branch
      );
      debug('files %j', data.data.files);

      const committedFiles = data.data.files;
      const schemaChanges = committedFiles.filter(
        s => s.filename.indexOf('_schemas/') === 0
      );
      const collectionChanges = committedFiles.filter(s =>
        /\.(html|md|markdown)$/i.test(s.filename)
      );
      if (schemaChanges.length === 0) {
        return Bluebird.resolve(collectionChanges);
      }
      // process schema changes first
      schemaUpdated = true;
      return Bluebird.map(
        schemaChanges,
        schema => {
          let schemaId = /^_schemas\/(\w+)\.json$/gi.exec(schema.filename)[1];
          schemaArray = schemaArray.filter(s => s.jekyll.id !== schemaId);
          if (schema.status !== 'removed') {
            return github
              .getContents(branch, schema.filename, true)
              .then(data => {
                if (schema.status === 'added') {
                  newSchemas.push(data.data);
                }
                schemaArray.push(data.data);
              });
          }
        },
        {
          concurrency: 5
        }
      ).then(() => {
        return collectionChanges;
      });
    })
    .then(collectionChanges => {
      return Bluebird.map(
        collectionChanges,
        f => {
          const { filename } = f;

          const collectionType = getCollectionType(schemaArray, filename);

          if (collectionType === '') {
            return Bluebird.resolve();
          }

          if (f.status === 'removed') {
            return removeEntryOfRepo({
              filename: filename,
              repoIndex
            }).then(removed => {
              if (!removed) return;
              updatedContent.collections.removed.push(removed);
            });
          } else {
            // consider added as modified
            return upsertEntryOfRepo({
              github,
              branch,
              collectionType,
              filename: filename,
              repoIndex
            }).then(modified => {
              if (!modified) return;
              updatedContent.collections.modified.push(modified);
            });
          }
        },
        {
          concurrency: 5
        }
      );
    })
    .then(() => {
      if (schemaUpdated) {
        repoIndex.schemas = JSON.stringify(schemaArray);
      }
      // update the tipCommitSha
      repoIndex.tipCommitSha = tipCommitSha;
      repoIndex.updated = Date();
      return repoIndex.save();
    })
    .then(() => {
      dispose();
      if (schemaUpdated) {
        updatedContent.schemas = schemaArray;
      }
      return updatedContent;
    })
    .catch(err => {
      if (err !== 'break') throw err;

      dispose();
      return updatedContent;
    });
}

/**
 * delete the entry instance and remove it from the repo
 *
 * @param {string} filename
 * @param {object} repoIndex - mongoose RepoIndex doc instance
 * @return {Promise} - resolve to undefined or { _id: string, path: string }
 */
function removeEntryOfRepo({ filename, repoIndex }) {
  return RepoFileEntry.findOne({
    path: filename,
    repoBranch: repoIndex
  }).then(entryInst => {
    if (!entryInst) return;

    repoIndex.collections.remove(entryInst._id);
    const removed = {
      _id: entryInst._id,
      path: entryInst.path
    };
    return repoIndex
      .save()
      .then(() => RepoFileEntry.find({ _id: entryInst._id }).remove().exec())
      .then(() => removed);
  });
}

/**
 * update or add a entry to a repo
 * will call github api to fetch file content and commit history
 *
 * @param {object} github - github-api instance
 * @param {string} branch
 * @param {string} collectionType - e.g. `post`
 * @param {string} filename
 * @param {object} repoIndex - mongoose RepoIndex doc instance
 * @return {Promise} - resolve to undefined or a entry
 */
function upsertEntryOfRepo({
  github,
  branch,
  collectionType,
  filename,
  repoIndex
}) {
  return Promise.all([
    github.getContents(branch, filename, true),
    github.listCommits({ sha: branch, path: filename })
  ]).then(([data, commits]) => {
    const content = data.data;
    const lastCommit = commits.data[0];

    const cond = {
      path: filename,
      repoBranch: repoIndex
    };
    const entry = {
      collectionType,
      content,
      lastCommitSha: lastCommit.sha,
      lastUpdatedAt: lastCommit.commit.committer.date,
      lastUpdatedBy: lastCommit.commit.committer.name,
      path: filename,
      repoBranch: repoIndex
    };
    const opt = { upsert: true, new: true };
    return RepoFileEntry.findOneAndUpdate(cond, entry, opt).then(entryInst => {
      if (!entryInst) {
        debug('faild to upsert file entry: %o', cond);
        // TODO better throw here
        return Promise.reject({
          status: 500,
          response: {
            data: {
              message: 'unable to create file entry instance in database'
              // TODO errorCode
            }
          }
        });
      }

      repoIndex.collections.addToSet(entryInst);
      const modified = lodash.pick(entry, [
        'collectionType',
        'content',
        'lastCommitSha',
        'lastUpdatedAt',
        'lastUpdatedBy',
        'path'
      ]);

      return repoIndex.save().then(() => modified);
    });
  });
}

const refreshIndexAndSave = (req, res) => {
  var repo = req.githubRepo;
  var branch = req.query.branch || 'master';
  var repoFullname = req.get('X-REPO-OWNER') + '/' + req.get('X-REPO-NAME');
  debug(`refreshIndexAndSave - ${repoFullname}`);
  var jobKey = repoFullname + '_' + branch;

  // avoid doing multiple refresh to same repo branch at same time
  if (RUNNING_JOBS[jobKey]) {<commit_msg>Using all lowercase letters in repo fullname<commit_after>import GithubAPI from 'github-api';
import _ from 'lodash';
import Bluebird from 'bluebird';
import mongoose from 'mongoose';
import _debug from 'debug';

const ObjectId = mongoose.Types.ObjectId;
const debug = _debug('jekyllpro-cms:repository');
mongoose.Promise = Bluebird;
const RepoFileEntry = mongoose.model('RepoFileEntry');
const lodash = _; // TODO

import {
  TaskQueue,
  getCollectionFiles,
  getCollectionType,
  getLangFromConfigYaml
} from './utils';
import { RepoIndex, RepoAccessToken } from './database';
import { hookConfig } from './webhook';

const RUNNING_JOBS = {};

const cb = (error, result, request) => {
  console.log(error);
  console.log(result);
};

/**
 * middleware
 * attach github-api instance to `req` - req.githubRepo
 */
const requireGithubAPI = (req, res, next) => {
  if (req.githubRepo) {
    return next();
  }
  const repoOwner = req.get('X-REPO-OWNER').toLowerCase();
  const repoName = req.get('X-REPO-NAME').toLowerCase();

  // attach repo info to request
  const repo = {
    owner: repoOwner,
    name: repoName,
    fullName: `${repoOwner}/${repoName}`
  };
  req.repo = repo;

  if (!repoName || !repoOwner) {
    res.status(401).send({ message: 'repository undefined' });
  }
  var ak =
    req.get('X-TOKEN') || req.user.accessToken || req.user._json.accessToken;
  var github = new GithubAPI({
    token: ak
  });
  req.githubRepo = github.getRepo(repoOwner, repoName);
  next();
};

const getRepoDetails = (req, res, next) => {
  var repo = req.githubRepo;

  repo
    .getDetails()
    .then(data => {
      // console.log(data)
      var details = _.pick(data.data, [
        'name',
        'full_name',
        'description',
        'private',
        'url',
        'html_url',
        'default_branch',
        'created_at',
        'updated_at',
        'pushed_at',
        'permissions'
      ]);
      details.owner = _.pick(data.data.owner, ['login', 'avatar_url', 'type']);
      res.status(200).json(details);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.response.status).json(err.response.data);
    });
};

const getRepoContent = (req, res, next) => {
  var repo = req.githubRepo;
  var { branch, path, raw } = req.query;

  repo
    .getContents(branch, path, raw)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const writeRepoFile = (req, res, next) => {
  var repo = req.githubRepo;
  var newFile = req.body;
  if (!newFile.options) {
    newFile.options = { encode: true };
  }
  repo
    .writeFile(
      newFile.branch,
      newFile.path,
      newFile.content,
      newFile.message,
      newFile.options,
      cb
    )
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const deleteRepoFile = (req, res, next) => {
  var repo = req.githubRepo;
  var toBeDeleted = req.body;

  repo
    .deleteFile(toBeDeleted.branch, toBeDeleted.path, cb)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const listBranches = (req, res, next) => {
  var repo = req.githubRepo;
  var { branch } = req.query;

  var nextPromise = branch ? repo.getBranch(branch) : repo.listBranches();

  return nextPromise
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res
        .status(err.status || err.response.status || err.response.status)
        .json(err.response.data);
    });
};

const createBranches = (req, res, next) => {
  var repo = req.githubRepo;
  var formData = req.body;

  repo
    .createBranch(formData.oldBranch, formData.newBranch)
    .then(data => {
      // console.log(data)
      res.status(200).json(data.data);
    })
    .catch(err => {
      // console.log(err)
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const getBranchSchema = (req, res, next) => {
  var repo = req.githubRepo;
  var { ref, path } = req.query;
  path = path || '_schema';

  repo
    .getContents(ref, path)
    .then(data => {
      let schemaFiles = data.data
        .filter(item => {
          // filter out folder files
          return item.type === 'file';
        })
        .map(item => {
          return repo.getContents(ref, item.path, true).then(data => {
            return { name: item.name, data: data.data };
          });
        });

      return Promise.all(schemaFiles).then(results => {
        res.status(200).json(results);
      });
    })
    .catch(err => {
      console.log(err);
      res.status(err.status || err.response.status).json(err.response.data);
    });
};

const getRepoBranchIndex = (req, res, next) => {
  var repo = req.githubRepo;
  var branch = req.query.branch || 'master';
  var refreshIndex = req.query.refresh === true || req.query.refresh === 'true';
  const repoFullname = req.repo.fullName;

  // update access token in db, which can be used to run the webhook service
  RepoAccessToken.findOneAndUpdate(
    {
      repository: repoFullname
    },
    {
      repository: repoFullname,
      accessToken: req.user.accessToken,
      updatedBy: req.user.login,
      updated: Date()
    },
    {
      upsert: true
    },
    err => {
      if (err) console.log(err);
    }
  );

  if (refreshIndex) {
    // set flag to clean old db record later if a new index cannot be built
    req.purgeDb = true;
    return next();
  }

  RepoIndex.findByRepoInfo(repoFullname, branch, (err, record) => {
    if (err) {
      console.log(err);
      return next();
    }
    if (!record) {
      return next();
    }
    debug(
      'get database record:',
      record.repository,
      record.branch,
      record.updated,
      record.tipCommitSha
    );
    // some repo branches might have legacy index data built when it didn't have schemas.
    // only return record if it has schemas.

    // collections now should be an array though unpopulated
    if (Array.isArray(record.collections) && record.tipCommitSha) {
      record
        .populate({
          path: 'collections',
          options: {
            sort: {
              lastUpdatedAt: 'desc'
            }
          }
        })
        .execPopulate()
        .then(record => {
          // @see mongoose doc
          // http://mongoosejs.com/docs/api.html#document_Document-execPopulate
          debug('record id is %s', record._id);
          let data = {
            updated: record.updated,
            collections: record.collections,
            schemas: JSON.parse(record.schemas),
            config: JSON.parse(record.config)
          };
          return res.status(200).json(data);
        });
    } else {
      debug('record invalid');
      // set flag to clean old db record later if a new index cannot be built
      req.purgeDb = true;
      // next middleware should be refreshIndexAndSave()
      return next();
    }
  });
};

function getRepoBranchUpdatedCollections(req, res) {
  const repo = req.githubRepo;
  const branch = req.query.branch || 'master';
  const repoFullname = req.repo.fullName;
  // TODO
  // currently refreshIndexIncremental will always resolve
  // but we need to figure
  return refreshIndexIncremental({
    github: repo,
    repoFullname,
    branch
  })
    .then(updatedContent => res.status(200).json(updatedContent))
    .catch(
      err =>
        console.log(err) ||
        res.status(err.status || err.response.status).json(err.response.data)
    );
}

/**
 * @return {Promise<collections>} - {collections: { modified: Array, removed: Array }}
 */
function refreshIndexIncremental({ github, repoFullname, branch }) {
  debug(`refreshIndexIncremental ${repoFullname}:${branch}`);
  let repoIndex = null;
  let schemaArray = null;
  let schemaUpdated = false;
  let tipCommitSha = null;
  let newSchemas = [];
  const updatedContent = {
    collections: {
      modified: [], // consider added as modified
      removed: []
    }
  };

  // throttle
  const jobKey = `refreshIndexIncremental:${repoFullname}:${branch}`;
  if (RUNNING_JOBS[jobKey]) {
    debug('another refreshIndexIncremental task is running, returning');
    return Promise.resolve(updatedContent);
  }
  RUNNING_JOBS[jobKey] = true;
  const dispose = () => {
    delete RUNNING_JOBS[jobKey];
  };

  // get last commit
  return Promise.all([
    RepoIndex.findOne({ repository: repoFullname, branch }),
    github.getRef(`heads/${branch}`)
  ])
    .then(([_repoIndex, refData]) => {
      if (!_repoIndex.tipCommitSha) {
        debug(
          'Error: no tipCommitSha on %s repoIndex %s',
          repoFullname,
          _repoIndex._id
        );
        throw 'break';
      }

      tipCommitSha = lodash.get(refData, 'data.object.sha');
      // can not get the commit sha
      if (typeof tipCommitSha !== 'string') throw 'break';
      // the commit sha is same as the one in database
      if (tipCommitSha === _repoIndex.tipCommitSha) throw 'break';

      debug(
        `tip commit: db is ${_repoIndex.tipCommitSha} remote is ${tipCommitSha}`
      );
      repoIndex = _repoIndex;
      schemaArray = JSON.parse(repoIndex.schemas);

      // doc https://developer.github.com/v3/repos/commits/#compare-two-commits
      return github.compareBranches(repoIndex.tipCommitSha, branch);
    })
    .then(data => {
      debug(
        '%d commits between %s and %s',
        data.data.total_commits,
        repoIndex.tipCommitSha,
        branch
      );
      debug('files %j', data.data.files);

      const committedFiles = data.data.files;
      const schemaChanges = committedFiles.filter(
        s => s.filename.indexOf('_schemas/') === 0
      );
      const collectionChanges = committedFiles.filter(s =>
        /\.(html|md|markdown)$/i.test(s.filename)
      );
      if (schemaChanges.length === 0) {
        return Bluebird.resolve(collectionChanges);
      }
      // process schema changes first
      schemaUpdated = true;
      return Bluebird.map(
        schemaChanges,
        schema => {
          let schemaId = /^_schemas\/(\w+)\.json$/gi.exec(schema.filename)[1];
          schemaArray = schemaArray.filter(s => s.jekyll.id !== schemaId);
          if (schema.status !== 'removed') {
            return github
              .getContents(branch, schema.filename, true)
              .then(data => {
                if (schema.status === 'added') {
                  newSchemas.push(data.data);
                }
                schemaArray.push(data.data);
              });
          }
        },
        {
          concurrency: 5
        }
      ).then(() => {
        return collectionChanges;
      });
    })
    .then(collectionChanges => {
      return Bluebird.map(
        collectionChanges,
        f => {
          const { filename } = f;

          const collectionType = getCollectionType(schemaArray, filename);

          if (collectionType === '') {
            return Bluebird.resolve();
          }

          if (f.status === 'removed') {
            return removeEntryOfRepo({
              filename: filename,
              repoIndex
            }).then(removed => {
              if (!removed) return;
              updatedContent.collections.removed.push(removed);
            });
          } else {
            // consider added as modified
            return upsertEntryOfRepo({
              github,
              branch,
              collectionType,
              filename: filename,
              repoIndex
            }).then(modified => {
              if (!modified) return;
              updatedContent.collections.modified.push(modified);
            });
          }
        },
        {
          concurrency: 5
        }
      );
    })
    .then(() => {
      if (schemaUpdated) {
        repoIndex.schemas = JSON.stringify(schemaArray);
      }
      // update the tipCommitSha
      repoIndex.tipCommitSha = tipCommitSha;
      repoIndex.updated = Date();
      return repoIndex.save();
    })
    .then(() => {
      dispose();
      if (schemaUpdated) {
        updatedContent.schemas = schemaArray;
      }
      return updatedContent;
    })
    .catch(err => {
      if (err !== 'break') throw err;

      dispose();
      return updatedContent;
    });
}

/**
 * delete the entry instance and remove it from the repo
 *
 * @param {string} filename
 * @param {object} repoIndex - mongoose RepoIndex doc instance
 * @return {Promise} - resolve to undefined or { _id: string, path: string }
 */
function removeEntryOfRepo({ filename, repoIndex }) {
  return RepoFileEntry.findOne({
    path: filename,
    repoBranch: repoIndex
  }).then(entryInst => {
    if (!entryInst) return;

    repoIndex.collections.remove(entryInst._id);
    const removed = {
      _id: entryInst._id,
      path: entryInst.path
    };
    return repoIndex
      .save()
      .then(() => RepoFileEntry.find({ _id: entryInst._id }).remove().exec())
      .then(() => removed);
  });
}

/**
 * update or add a entry to a repo
 * will call github api to fetch file content and commit history
 *
 * @param {object} github - github-api instance
 * @param {string} branch
 * @param {string} collectionType - e.g. `post`
 * @param {string} filename
 * @param {object} repoIndex - mongoose RepoIndex doc instance
 * @return {Promise} - resolve to undefined or a entry
 */
function upsertEntryOfRepo({
  github,
  branch,
  collectionType,
  filename,
  repoIndex
}) {
  return Promise.all([
    github.getContents(branch, filename, true),
    github.listCommits({ sha: branch, path: filename })
  ]).then(([data, commits]) => {
    const content = data.data;
    const lastCommit = commits.data[0];

    const cond = {
      path: filename,
      repoBranch: repoIndex
    };
    const entry = {
      collectionType,
      content,
      lastCommitSha: lastCommit.sha,
      lastUpdatedAt: lastCommit.commit.committer.date,
      lastUpdatedBy: lastCommit.commit.committer.name,
      path: filename,
      repoBranch: repoIndex
    };
    const opt = { upsert: true, new: true };
    return RepoFileEntry.findOneAndUpdate(cond, entry, opt).then(entryInst => {
      if (!entryInst) {
        debug('faild to upsert file entry: %o', cond);
        // TODO better throw here
        return Promise.reject({
          status: 500,
          response: {
            data: {
              message: 'unable to create file entry instance in database'
              // TODO errorCode
            }
          }
        });
      }

      repoIndex.collections.addToSet(entryInst);
      const modified = lodash.pick(entry, [
        'collectionType',
        'content',
        'lastCommitSha',
        'lastUpdatedAt',
        'lastUpdatedBy',
        'path'
      ]);

      return repoIndex.save().then(() => modified);
    });
  });
}

const refreshIndexAndSave = (req, res) => {
  var repo = req.githubRepo;
  var branch = req.query.branch || 'master';
  const repoFullname = req.repo.fullName;
  debug(`refreshIndexAndSave - ${repoFullname}`);
  var jobKey = repoFullname + '_' + branch;

  // avoid doing multiple refresh to same repo branch at same time
  if (RUNNING_JOBS[jobKey]) {
{'fraction_non_alphanumeric': 0.13850362027353177, 'fraction_numerical': 0.002574416733708769, 'mean_word_length': 2.2465524446301712, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 117, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1273334', 'n_tokens_mistral': 9554, 'n_tokens_neox': 9522, 'n_words': 2132}