<commit_before>filebeat:
  # List of prospectors to fetch data.
  prospectors:
    -
      paths:
        - /output/logs/crawl.log
        - /data/output/logs/crawl.log
      type: log
      document_type: heritrix
      fields:
        crawl_id: daily

shipper:
  name: heritrix

output:
  logstash:
    enabled: true

    hosts: ["monitrix:5044"]<commit_msg>Added monitrix type.<commit_after>filebeat:
  # List of prospectors to fetch data.
  prospectors:
    -
      paths:
        - /output/logs/crawl.log
        - /data/output/logs/crawl.log
      type: log
      document_type: heritrix
      crawl_id: daily

shipper:
  name: heritrix

output:
  logstash:
    enabled: true

    hosts: ["monitrix:5044"]
{'fraction_non_alphanumeric': 0.09845288326300984, 'fraction_numerical': 0.011251758087201125, 'mean_word_length': 2.192825112107623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5187004', 'n_tokens_mistral': 261, 'n_tokens_neox': 252, 'n_words': 54}