<commit_before>
var https = require('https')


function toQueryString (params) {
  return Object.keys(params)
    .map((name) => (name + '=' + params[name]))
    .join('&')
}

function getPath (api, params) {
  var path = '/api/v1/' + api + '.json'
  if (params) {
    path += '?' + toQueryString(params)
  }
  return path
}

exports.get = (api, params, done) => {
  if (typeof params === 'function') {
    done = params
    params = {}
  }
  https.request({
    hostname: 'rubygems.org',
    port: 443,
    path: getPath(api, params),
    method: 'GET'
  }, (res) => {
    var buff = ''
    res.on('data', (chunk) => {
      buff += chunk
    })
    .on('end', () => {
      res.raw = buff.toString()
      try {
        res.body = JSON.parse(res.raw)
      }
      catch (e) {
        var err = new Error(res.raw)
      }
      done(err, res)
    })
  })
  .on('error', done)
  .end()
}

exports.scrape = (api, done) => {
  https.request({
    hostname: 'rubygems.org',
    port: 443,<commit_msg>Minor refactoring in rubygems module<commit_after>
var https = require('https')


function toQueryString (qs) {
  return Object.keys(qs)
    .map((name) => (name + '=' + qs[name]))
    .join('&')
}

function getPath (api, qs) {
  var path = '/api/v1/' + api + '.json'
  if (qs) {
    path += '?' + toQueryString(qs)
  }
  return path
}

exports.get = (api, options, done) => {
  if (typeof options === 'function') {
    done = options
    options = {}
  }

  var opts = {
    hostname: 'rubygems.org',
    port: 443,
    method: 'GET',
    path: getPath(api, options.qs)
  }
  if (options.agent) {
    opts.agent = options.agent
  }

  var req = https.request(opts)

  req.on('response', (res) => {
    var buff = ''
    res
      .on('data', (chunk) => {
        buff += chunk
      })
      .on('end', () => {
        res.raw = buff.toString()
        try {
          res.body = JSON.parse(res.raw)
        }
        catch (e) {
          var err = new Error(res.raw)
        }
        done(err, res)
      })
  })

  req.on('error', done)
  req.end()
}

exports.scrape = (api, done) => {
  https.request({
    hostname: 'rubygems.org',
    port: 443,
{'fraction_non_alphanumeric': 0.1887494188749419, 'fraction_numerical': 0.006508600650860065, 'mean_word_length': 2.1370262390670556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2712851', 'n_tokens_mistral': 810, 'n_tokens_neox': 789, 'n_words': 175}