<commit_before>'use strict';
var axios = require('axios');
var pathToRegexp = require('path-to-regexp');
var _ = require('lodash');

function postRequest(url, item) {
  return axios.post(url, item)
    .then(function(res) {
      return res.data;
    }).catch(function(err) {
      console.error(err);
      throw new Error('cs-node-error');
    });
}

function batchGetRequest(url, path, items, options) {
  options = options || {};

  items = items !== undefined ? [].concat(items) : [];

  // filter empty items
  items = items.filter(function(item) {
    return !!item;
  });

  // group items by chunks
  var maxChunk = options.maxChunk || 50;
  var isString = !Array.isArray(items) || typeof items[0] !== 'object';

  if (isString) {
    items = _.chunk(items, maxChunk).map(function(items) {
      return items.join(',');
    });
  } else {
    items = _.chunk(items, maxChunk).map(function(items) {
      return items.reduce(function(sum, item) {
        Object.keys(item).forEach(function(key) {
          if (sum[key]) {
            sum[key] += ',' + item[key];
          } else {
            sum[key] = '' + item[key];
          }
        });
        return sum;
      }, {});
    });
  }

  return Promise.all(items.map(function(item) {
    var tokens = pathToRegexp.parse(path);
    var queryUrl;

    if (tokens.length > 1) {
      queryUrl = url + pathToRegexp.tokensToFunction(tokens)(item);
    } else {
      queryUrl = url + path + encodeURIComponent(item);
    }
    return getRequest(queryUrl, options);
  })).then(function(res) {
    return _.flatten(res);
  });
}

function getRequest(url, options) {
  options = options || {};
  var delayBeforeRetry = options.delayBeforeRetry !== undefined ? options.delayBeforeRetry : 1000;
  return request(url, options.retry || 100, delayBeforeRetry, options.params || {})
    .then(function(res) {
      return res.data;
    });
}

function request(url, retry, delayBeforeRetry, params) {<commit_msg>Update batchGetRequest<commit_after>'use strict';
var axios = require('axios');
var pathToRegexp = require('path-to-regexp');
var _ = require('lodash');

function postRequest(url, item) {
  return axios.post(url, item)
    .then(function(res) {
      return res.data;
    }).catch(function(err) {
      console.error(err);
      throw new Error('cs-node-error');
    });
}

function batchGetRequest(url, path, items, options) {
  options = options || {};

  items = items !== undefined ? [].concat(items) : [];

  // filter empty items
  items = items.filter(function(item) {
    return !!item;
  });

  // group items by chunks
  var maxChunk = options.maxChunk || 10;
  var isString = !Array.isArray(items) || typeof items[0] !== 'object';

  if (isString) {
    items = _.chunk(items, maxChunk).map(function(items) {
      return items.join(',');
    });
  } else {
    items = _.chunk(items, maxChunk).map(function(items) {
      return items.reduce(function(sum, item) {
        Object.keys(item).forEach(function(key) {
          if (sum[key]) {
            sum[key] += ',' + item[key];
          } else {
            sum[key] = '' + item[key];
          }
        });
        return sum;
      }, {});
    });
  }

  return items.reduce(function(promise, item) {
    return promise.then(function(result) {
      var tokens = pathToRegexp.parse(path);
      var queryUrl;
      if (tokens.length > 1) {
        queryUrl = url + pathToRegexp.tokensToFunction(tokens)(item);
      } else {
        queryUrl = url + path + encodeURIComponent(item);
      }
      return getRequest(queryUrl, options).then(Array.prototype.concat.bind(result));
    });
  }, Promise.resolve([])).then(function(res) {
    return res;
  });
}

function getRequest(url, options) {
  options = options || {};
  var delayBeforeRetry = options.delayBeforeRetry !== undefined ? options.delayBeforeRetry : 1000;
  return request(url, options.retry || 100, delayBeforeRetry, options.params || {})
    .then(function(res) {
      return res.data;
    });
}

function request(url, retry, delayBeforeRetry, params) {
{'fraction_non_alphanumeric': 0.1622222222222222, 'fraction_numerical': 0.005432098765432099, 'mean_word_length': 2.9521951219512195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1018974', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1276, 'n_words': 262}