<commit_before>const { RateLimiter } = require('limiter')
const { writeFile, ExtendableError } = require('./utils')

// We will limit all network requests to a maximum of 40 per 10 seconds.
// Note: Chromium will additionally limit the maximum number of concurrent
// open requests from an origin to 6 as of April 2017.
const fortyRequestsPer = 40
const tenSeconds = 10 * 1000 // time in milliseconds
let limiter = new RateLimiter(fortyRequestsPer, tenSeconds)

// Requests are rejected with NetworkError when the network connection has
// issues or the request times out.
class NetworkError extends ExtendableError {
  constructor (message, url) {
    super(message)
    this.url = url<commit_msg>Allow 80 requests per 10 seconds<commit_after>const { RateLimiter } = require('limiter')
const { writeFile, ExtendableError } = require('./utils')

// We will limit all network requests to a maximum of 80 per 10 seconds.
// Note: Chromium will additionally limit the maximum number of concurrent
// open requests from an origin to 6 as of April 2017.
const eightyRequestsPer = 80
const tenSeconds = 10 * 1000 // time in milliseconds
let limiter = new RateLimiter(eightyRequestsPer, tenSeconds)

// Requests are rejected with NetworkError when the network connection has
// issues or the request times out.
class NetworkError extends ExtendableError {
  constructor (message, url) {
    super(message)
    this.url = url
{'fraction_non_alphanumeric': 0.07339449541284404, 'fraction_numerical': 0.026817219477769938, 'mean_word_length': 4.291044776119403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1418810', 'n_tokens_mistral': 401, 'n_tokens_neox': 359, 'n_words': 170}