<commit_before>var request = require('request')
var fs = require('fs')
var writemarkdown = require('./writemarkdown.js')
var writehtml = require('./writehtml.js')

var base = 'https://api.github.com'
var headers = {"user-agent": "offline-issues module",}

var issueData = []
var counter = 1
var done = ''

module.exports = function getIssues(token, options, cb) {
  headers["Authorization"] = 'token ' + token.token
  if (options._.length === 0 && options.html) {
    return writehtml(cb)
  }
  if (options._.length === 0) return cb(null, "No repository given.")
  parseRepo(options, cb)
}

function parseRepo(options, cb) {
  done = options._.length
  options.repos = []

  options._.forEach(function(repo) {
    var repoDetails = {}
    repoDetails.full = repo
    var userAndRepo = repo.split('/')
    repoDetails.user = userAndRepo[0]
    if (userAndRepo[1].indexOf('#') >= 0) {
      var repoAndIssue = userAndRepo[1].split('#')
      repoDetails.name = repoAndIssue[0]
      repoDetails.issue = repoAndIssue[1]
    } else { repoDetails.name = userAndRepo[1] }
    options.repos.push(repoDetails)
  })
  options.repos.forEach(function(repo) {
    getIssue(repo, cb)
  })
}

function getIssue(repo, cb) {
  var url = base + '/repos/' + repo.user + '/' + repo.name + '/issues/' + repo.issue
  request(url, {json: true, headers: headers}, function(err, resp, body) {
    if (err) return cb(err, "Error in request for issue.")
    loadIssue(body, repo, cb)
  })
}

function loadIssue(body, repo, cb) {
  var issue = {}

  issue.id = body.id
  issue.url = body.html_url
  issue.title = body.title
  issue.created_by = body.user.login || body.head.user.login
  issue.created_at = new Date(body.created_at).toLocaleDateString()
  issue.body = body.body
  issue.state = body.state
  issue.comments = []
  issue.quicklink = repo.full

  getComments(issue, repo, cb)
}

function getComments(issue, repo, cb) {
  var url = base + '/repos/' + repo.user + '/' + repo.name + '/issues/' + repo.issue + '/comments'

  request(url, {json: true, headers: headers}, function(err, resp, body) {
    if (err) return cb(err, "Error in request for comments.")

    issue.comments = body
    issue.comments.forEach(function(comment) {
      comment.created_at = new Date(comment.created_at).toLocaleDateString()
    })
    issueData.push(issue)
<commit_msg>support 'all'<commit_after>var request = require('request')
var fs = require('fs')
var writemarkdown = require('./writemarkdown.js')
var writehtml = require('./writehtml.js')

var base = 'https://api.github.com'
var headers = {"user-agent": "offline-issues module",}

var issueData = []
var counter = 1
var done = ''

module.exports = function getIssues(token, options, cb) {
  headers["Authorization"] = 'token ' + token.token
  if (options._.length === 0 && options.html) {
    return writehtml(cb)
  }
  if (options._.length === 0) return cb(null, "No repository given.")
  parseRepo(options, cb)
}

function parseRepo(options, cb) {
  done = options._.length
  options.repos = []

  options._.forEach(function(repo) {
    var repoDetails = {}
    repoDetails.full = repo
    var userAndRepo = repo.split('/')
    repoDetails.user = userAndRepo[0]
    if (userAndRepo[1].indexOf('#') >= 0) {
      var repoAndIssue = userAndRepo[1].split('#')
      repoDetails.name = repoAndIssue[0]
      repoDetails.issue = repoAndIssue[1]
    } else {
      repoDetails.name = userAndRepo[1]
      repoDetails.issue = 'all'
    }
    options.repos.push(repoDetails)
    console.log(repoDetails)
  })
  options.repos.forEach(function(repo) {
    getIssue(repo, cb)
  })
}

function getIssue(repo, cb) {
  var url = base + '/repos/' + repo.user + '/' + repo.name + '/issues'
  if (repo.issue === 'all') {
    url = url + '?state=all'
  } else url = url + '/' + repo.issue
  console.log(url)
  request(url, {json: true, headers: headers}, function(err, resp, body) {
    if (err) return cb(err, "Error in request for issue.")
    if (repo.issue === 'all') {
      return body.forEach(function(issue) {
        loadIssue(issue, repo, cb)
      })
    }
    loadIssue(body, repo, cb)
  })
}

function loadIssue(body, repo, cb) {
  var issue = {}

  issue.id = body.id
  issue.url = body.html_url
  issue.title = body.title
  issue.created_by = body.user.login || body.head.user.login
  issue.created_at = new Date(body.created_at).toLocaleDateString()
  issue.body = body.body
  issue.state = body.state
  issue.comments = []
  issue.quicklink = repo.full
  issue.comments_url = body.comments_url

  getComments(issue, repo, cb)
}

function getComments(issue, repo, cb) {
  // console.log('get coms', issue)
  if (repo.issue === 'all') {
    var url = issue.comments_url
  } else {
    var url = base + '/repos/' + repo.user + '/' + repo.name + '/issues/' + repo.issue + '/comments'
  }
  request(url, {json: true, headers: headers}, function(err, resp, body) {
    if (err) return cb(err, "Error in request for comments.")

    issue.comments = body
    issue.comments.forEach(function(comment) {
      comment.created_at = new Date(comment.created_at).toLocaleDateString()
    })
    issueData.push(issue)

{'fraction_non_alphanumeric': 0.1582537517053206, 'fraction_numerical': 0.0038978756577665173, 'mean_word_length': 3.435609334485739, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5234124', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1711, 'n_words': 394}