<commit_before>tika:
  url: http://localhost:8231  # ipfs-tika endpoint URL, also TIKA_URL in env
  timeout: 5m  # ipfs-tika request timeout
  max_size: 50MB  # Don't attempt to get metadata for files over this size
ipfs:
  api_url: localhost:5001  # IPFS API endpoint, also IPFS_API_URL in env
  timeout: 6m  # Timeout for IPFS gateway HTTPS requests
elasticsearch:
  url: http://localhost:9200  # Also ELASTICSEARCH_URL in env
amqp:
  url: amqp://guest:guest@localhost:5672/  # Also AMQP_URL in env
crawler:
  retry_wait: 2s  # wait time between retries of failed requests
  hash_wait: 100ms  # Time between launching workers
  file_wait: 100ms
  partial_size: 256KB  # Size for partial items - this is the default chunker block size
  hash_workers: 140
  file_workers: 120
<commit_msg>Half number of crawler workers (performance test).<commit_after>tika:
  url: http://localhost:8231  # ipfs-tika endpoint URL, also TIKA_URL in env
  timeout: 5m  # ipfs-tika request timeout
  max_size: 50MB  # Don't attempt to get metadata for files over this size
ipfs:
  api_url: localhost:5001  # IPFS API endpoint, also IPFS_API_URL in env
  timeout: 6m  # Timeout for IPFS gateway HTTPS requests
elasticsearch:
  url: http://localhost:9200  # Also ELASTICSEARCH_URL in env
amqp:
  url: amqp://guest:guest@localhost:5672/  # Also AMQP_URL in env
crawler:
  retry_wait: 2s  # wait time between retries of failed requests
  hash_wait: 100ms  # Time between launching workers
  file_wait: 100ms
  partial_size: 256KB  # Size for partial items - this is the default chunker block size
  hash_workers: 70
  file_workers: 60

{'fraction_non_alphanumeric': 0.06765983860955928, 'fraction_numerical': 0.043451272501551834, 'mean_word_length': 4.2, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3304427', 'n_tokens_mistral': 602, 'n_tokens_neox': 535, 'n_words': 194}