<commit_before>'use strict';

var hyperquest       =  require('hyperquest')
  , xtendUrl         =  require('extend-url')
  , through          =  require('through')
  , Stream           =  require('stream')
  , parseLinkHeader  =  require('parse-link-header')
  , withPagingParams =  require('./lib/with-paging-params')
  ;

var setImmediate = setImmediate || function (fn) { setTimeout(fn, 0) };

function writeStream() {
  var s = new Stream();
  s.readable = true;
  return s;
}

function nextPage(opts, cb) {
  
  function onresponse (err, res) {
    if (err) return cb(err);  
    var body = '';

    res.pipe(through(write, end));

    function write (d) { body += d }

    function end () {
      if (/^[45]\d\d/.test(res.statusCode)) return cb(body);

      if (opts.json) {
        try {
          body = JSON.parse(body);
        } catch (e) {
          return cb(e);
        }
      }

      cb(null, { headers: res.headers, statusCode: res.statusCode, body: body });
    }
  }

  hyperquest(opts, onresponse);
}

function getPage(which, links) {
  if (!links || !links[which] || !links[which].page) return null;

  var n = parseInt(links[which].page, 10);
  return isNaN(n) ? null: n;
}
var getLastPage = getPage.bind(0, 'last');
var getNextPage = getPage.bind(0, 'next');

function initGetPages(opts, limit, startPage, cb) {
  var stream = typeof cb !== 'function' ? writeStream() : null;

  function data (acc, res) {
    if (stream) stream.emit('data', JSON.stringify(res));
    else        acc.push(res);
  }<commit_msg>no longer parsing empty body<commit_after>'use strict';

var hyperquest       =  require('hyperquest')
  , xtendUrl         =  require('extend-url')
  , through          =  require('through')
  , Stream           =  require('stream')
  , parseLinkHeader  =  require('parse-link-header')
  , withPagingParams =  require('./lib/with-paging-params')
  ;

var setImmediate = setImmediate || function (fn) { setTimeout(fn, 0) };

function writeStream() {
  var s = new Stream();
  s.readable = true;
  return s;
}

function nextPage(opts, cb) {
  
  function onresponse (err, res) {
    if (err) return cb(err);  
    var body = '';

    res.pipe(through(write, end));

    function write (d) { body += d }

    function end () {
      if (/^[45]\d\d/.test(res.statusCode)) return cb(body);

      if (body.length && opts.json) {
        try {
          body = JSON.parse(body);
        } catch (e) {
          return cb(e);
        }
      }

      cb(null, { headers: res.headers, statusCode: res.statusCode, body: body });
    }
  }

  hyperquest(opts, onresponse);
}

function getPage(which, links) {
  if (!links || !links[which] || !links[which].page) return null;

  var n = parseInt(links[which].page, 10);
  return isNaN(n) ? null: n;
}
var getLastPage = getPage.bind(0, 'last');
var getNextPage = getPage.bind(0, 'next');

function initGetPages(opts, limit, startPage, cb) {
  var stream = typeof cb !== 'function' ? writeStream() : null;

  function data (acc, res) {
    if (stream) stream.emit('data', JSON.stringify(res));
    else        acc.push(res);
  }
{'fraction_non_alphanumeric': 0.16086395873629916, 'fraction_numerical': 0.004513217279174726, 'mean_word_length': 2.774939172749392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6210651', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1027, 'n_words': 251}