<commit_before># configuration for Elasticsearch resources
  
config:
  # ES indexer bolt
  es.indexer.addresses: "localhost:9300"
  es.indexer.index.name: "index"
  es.indexer.doc.type: "doc"
  es.indexer.create: false
  es.indexer.settings:
    cluster.name: "elasticsearch"
  
  # ES metricsConsumer
  es.metrics.addresses: "localhost:9300"
  es.metrics.index.name: "metrics"
  es.metrics.doc.type: "datapoint"
  es.metrics.settings:
    cluster.name: "elasticsearch"
  
  # ES spout and persistence bolt
  es.status.addresses: "localhost:9300"
  es.status.index.name: "status"
  es.status.doc.type: "status"
  # the routing is done on the value of 'partition.url.mode'
  es.status.routing: true
  # stores the value used for the routing as a separate field
  # needed by the spout implementations
  es.status.routing.fieldname: "metadata.hostname"
  es.status.bulkActions: 500
  es.status.flushInterval: "5s"
  es.status.concurrentRequests: 1
  es.status.settings:
    cluster.name: "elasticsearch"
  
  ################
  # spout config #
  ################

  # time in secs for which the URLs will be considered for fetching after a ack of fail
  es.status.ttl.purgatory: 30
  
  # Min time (in msecs) to allow between 2 successive queries to ES
  es.status.min.delay.queries: 1000
   
  es.status.max.buckets: 250
  es.status.max.urls.per.bucket: 4
  # field to group the URLs into buckets
  es.status.bucket.field: "metadata.hostname"
  # field to sort the URLs within a bucket
  es.status.bucket.sort.field: "nextFetchDate"
  # field to sort the buckets
  es.status.global.sort.field: "nextFetchDate"
  
  # CollapsingSpout : limits the deep paging by resetting the start offset for the ES query 
  es.status.max.start.offset: 500
  
  # AggregationSpout : sampling improves the performance on large crawls
  es.status.sample: true
  
  # es.status.recentDate.increase: 10
  # es.status.recentDate.min.gap: 2

  topology.metrics.consumer.register:
       - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
         parallelism.hint: 1
         #whitelist:
         #  - "fetcher_counter"
         #  - "fetcher_average.bytes_fetched"
         #blacklist:<commit_msg>reset.fetchdate.after: 180<commit_after># configuration for Elasticsearch resources
  
config:
  # ES indexer bolt
  es.indexer.addresses: "localhost:9300"
  es.indexer.index.name: "index"
  es.indexer.doc.type: "doc"
  es.indexer.create: false
  es.indexer.settings:
    cluster.name: "elasticsearch"
  
  # ES metricsConsumer
  es.metrics.addresses: "localhost:9300"
  es.metrics.index.name: "metrics"
  es.metrics.doc.type: "datapoint"
  es.metrics.settings:
    cluster.name: "elasticsearch"
  
  # ES spout and persistence bolt
  es.status.addresses: "localhost:9300"
  es.status.index.name: "status"
  es.status.doc.type: "status"
  # the routing is done on the value of 'partition.url.mode'
  es.status.routing: true
  # stores the value used for the routing as a separate field
  # needed by the spout implementations
  es.status.routing.fieldname: "metadata.hostname"
  es.status.bulkActions: 500
  es.status.flushInterval: "5s"
  es.status.concurrentRequests: 1
  es.status.settings:
    cluster.name: "elasticsearch"
  
  ################
  # spout config #
  ################

  # time in secs for which the URLs will be considered for fetching after a ack of fail
  es.status.ttl.purgatory: 30
  
  # Min time (in msecs) to allow between 2 successive queries to ES
  es.status.min.delay.queries: 1000
   
  es.status.max.buckets: 250
  es.status.max.urls.per.bucket: 4
  # field to group the URLs into buckets
  es.status.bucket.field: "metadata.hostname"
  # field to sort the URLs within a bucket
  es.status.bucket.sort.field: "nextFetchDate"
  # field to sort the buckets
  es.status.global.sort.field: "nextFetchDate"
  
  # Delay since previous query date (in secs) after which the nextFetchDate value will be reset
  es.status.reset.fetchdate.after: 180
  
  # CollapsingSpout : limits the deep paging by resetting the start offset for the ES query 
  es.status.max.start.offset: 500
  
  # AggregationSpout : sampling improves the performance on large crawls
  es.status.sample: true
  
  # es.status.recentDate.increase: 10
  # es.status.recentDate.min.gap: 2

  topology.metrics.consumer.register:
       - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
         parallelism.hint: 1
         #whitelist:
         #  - "fetcher_counter"
         #  - "fetcher_average.bytes_fetched"
         #blacklist:
{'fraction_non_alphanumeric': 0.11028283271212454, 'fraction_numerical': 0.016663012497259374, 'mean_word_length': 3.4162633107454017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941532', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1487, 'n_words': 385}