<commit_before>var async = require('async');
var GitHubApi = require('github');
var toTrack = require(".././to_track");
var data = require("./data");

/**
 * createGithubClient()
 * @return {GitHubApi}
 */
function createGithubClient() {
  var githubClient = new GitHubApi({
    version: '3.0.0',
    protocol: 'https'
  });
  githubClient.authenticate({
    type: 'basic',
    username: process.env.GITHUB_USERNAME,
    password: <PASSWORD>
  });
  return githubClient;
}

/**
 * A util
 */
function truthy(o) {
  if (o) {
    return true;
  }
  return false;
}

/**
 * Extract the values we want from all the data available in the API
 * @param  {JSON} ghRes, a single respsonse from the github API
 * @param  {String} org, the github org name
 * @return {Array}
 */
function getSelectedRepoValues(ghRes, org) {
  var arr = [];
  if (ghRes) {
    for (var i = 0; i < ghRes.length; i++) {
      var r = ghRes[i];
      if (r.name && r.updated_at) {
        arr.push({
          org: org,
          name: r.name,
          updated_at: r.updated_at,
          has_issues: r.has_issues,
          default_branch: r.default_branch
        });
      } else {
        console.log('FUNNY BUSINESS:', r);
      }
    }
  }
  return arr;
}

/**
 * Get a list of the repos linked to an org
 * @param  {String}   org      github org name
 * @param  {Function} callback
 * @return {Object}
 */
function getOrgRepos(org, callback) {
  var repos = [];
  var githubClient = createGithubClient();

  // The options msg we send to the client http://mikedeboer.github.io/node-github/#repos.prototype.getFromOrg
  var msg = {
    org: org,
    type: 'public',
    per_page: 100
  };

  // To see the data from github: curl -i https://api.github.com/orgs/mozilla/repos?per_page=1
  githubClient.repos.getFromOrg(msg, function gotFromOrg(err, res) {
    if (err) {
      console.log(err);
    }
    // this has loaded the first page of results
    // get the values we want out of this response
    repos = repos.concat(getSelectedRepoValues(res, org));

    // setup variables to use in the whilst loop below
    var ghResult = res;
    var hasNextPage = truthy(githubClient.hasNextPage(res));

    // now we work through any remaining pages
    async.whilst(
      function test() {
        return hasNextPage;
      },
      function doThis(callback) {
        githubClient.getNextPage(ghResult, function gotNextPage(err, res) {
          // get the values we want out of this response
          repos = repos.concat(getSelectedRepoValues(res));

          // update the variables used in the whilst logic
          ghResult = res;
          hasNextPage = truthy(githubClient.hasNextPage(res));

          callback(null);
        });
      },
      function done(err) {
        callback(null, repos);
      });
  });
}

/**
 * Save commits given to use from Gitub
 * @param  {JSON}   ghRes - the response from github
 * @param  {Object}   repo, reference object for which repo we're working on
 * @param  {Function} callback<commit_msg>bug fix<commit_after>var async = require('async');
var GitHubApi = require('github');
var toTrack = require(".././to_track");
var data = require("./data");

/**
 * createGithubClient()
 * @return {GitHubApi}
 */
function createGithubClient() {
  var githubClient = new GitHubApi({
    version: '3.0.0',
    protocol: 'https'
  });
  githubClient.authenticate({
    type: 'basic',
    username: process.env.GITHUB_USERNAME,
    password: <PASSWORD>
  });
  return githubClient;
}

/**
 * A util
 */
function truthy(o) {
  if (o) {
    return true;
  }
  return false;
}

/**
 * Extract the values we want from all the data available in the API
 * @param  {JSON} ghRes, a single respsonse from the github API
 * @param  {String} org, the github org name
 * @return {Array}
 */
function getSelectedRepoValues(ghRes, org) {
  var arr = [];
  if (ghRes) {
    for (var i = 0; i < ghRes.length; i++) {
      var r = ghRes[i];
      if (r.name && r.updated_at) {
        arr.push({
          org: org,
          name: r.name,
          updated_at: r.updated_at,
          has_issues: r.has_issues,
          default_branch: r.default_branch
        });
      } else {
        console.log('FUNNY BUSINESS:', r);
      }
    }
  }
  return arr;
}

/**
 * Get a list of the repos linked to an org
 * @param  {String}   org      github org name
 * @param  {Function} callback
 * @return {Object}
 */
function getOrgRepos(org, callback) {
  var repos = [];
  var githubClient = createGithubClient();

  // The options msg we send to the client http://mikedeboer.github.io/node-github/#repos.prototype.getFromOrg
  var msg = {
    org: org,
    type: 'public',
    per_page: 100
  };

  // To see the data from github: curl -i https://api.github.com/orgs/mozilla/repos?per_page=1
  githubClient.repos.getFromOrg(msg, function gotFromOrg(err, res) {
    if (err) {
      console.log(err);
    }
    // this has loaded the first page of results
    // get the values we want out of this response
    repos = repos.concat(getSelectedRepoValues(res, org));

    // setup variables to use in the whilst loop below
    var ghResult = res;
    var hasNextPage = truthy(githubClient.hasNextPage(res));

    // now we work through any remaining pages
    async.whilst(
      function test() {
        return hasNextPage;
      },
      function doThis(callback) {
        githubClient.getNextPage(ghResult, function gotNextPage(err, res) {
          // get the values we want out of this response
          repos = repos.concat(getSelectedRepoValues(res, org));

          // update the variables used in the whilst logic
          ghResult = res;
          hasNextPage = truthy(githubClient.hasNextPage(res));

          callback(null);
        });
      },
      function done(err) {
        callback(null, repos);
      });
  });
}

/**
 * Save commits given to use from Gitub
 * @param  {JSON}   ghRes - the response from github
 * @param  {Object}   repo, reference object for which repo we're working on
 * @param  {Function} callback
{'fraction_non_alphanumeric': 0.12373695544144443, 'fraction_numerical': 0.002650323008116614, 'mean_word_length': 2.549676660787772, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870089', 'n_tokens_mistral': 1972, 'n_tokens_neox': 1877, 'n_words': 569}