<commit_before>// @flow

import mixin from "merge-descriptors";
import EventEmitter from "events";
import SpiderQueue from "./queue/promise";
import SpiderRouter from "./routing/router";
import SpiderRequest from "./routing/request";
import SpiderResponse from "./routing/response";

/**
 * @param  {string}     base
 * @param  {Object}     options
 * @return {Object}
 */
function createCrawler(base: string, options: Object = {}): Object {
  if (typeof base !== "string") {
    throw new Error(`Base must be a string, not ${typeof base}`);
  }

  const config: Object = {
    interval: 250,
    concurrency: 10,
    ...options
  };

  const crawler: Object = {
    base: base,
    opts: config,
    req: SpiderRequest,
    res: SpiderResponse
  };

  // Glues all the components together:
  mixin(crawler, SpiderQueue, false);
  mixin(crawler, SpiderRouter, false);
  mixin(crawler, EventEmitter.prototype, false);

  return crawler;
}

module.exports = createCrawler;
module.exports.request = SpiderRequest;
module.exports.response = SpiderResponse;<commit_msg>Update default options to match new queue-promise API<commit_after>// @flow

import mixin from "merge-descriptors";
import EventEmitter from "events";
import SpiderQueue from "./queue/promise";
import SpiderRouter from "./routing/router";
import SpiderRequest from "./routing/request";
import SpiderResponse from "./routing/response";

/**
 * @param  {string}     base
 * @param  {Object}     options
 * @return {Object}
 */
function createCrawler(base: string, options: Object = {}): Object {
  if (typeof base !== "string") {
    throw new Error(`Base must be a string, not ${typeof base}`);
  }

  const config: Object = {
    interval: 250,
    concurrent: 10,
    ...options
  };

  const crawler: Object = {
    base: base,
    opts: config,
    req: SpiderRequest,
    res: SpiderResponse
  };

  // Glues all the components together:
  mixin(crawler, SpiderQueue, false);
  mixin(crawler, SpiderRouter, false);
  mixin(crawler, EventEmitter.prototype, false);

  return crawler;
}

module.exports = createCrawler;
module.exports.request = SpiderRequest;
module.exports.response = SpiderResponse;
{'fraction_non_alphanumeric': 0.1278264882325796, 'fraction_numerical': 0.0046146746654360865, 'mean_word_length': 3.850111856823266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591115', 'n_tokens_mistral': 675, 'n_tokens_neox': 648, 'n_words': 196}