<commit_before>{:consumer-properties {"zookeeper.connect"  "localhost:2181"
                       "group.id"           "bifrost"
                       "auto.offset.reset"  "smallest" ; we explicitly commit offsets once files have
                                                       ; been uploaded to s3 so no need for auto commit
                       "auto.commit.enable" "false"}
 :topic-blacklist     #{"sometopic"}
 :topic-whitelist     nil
 :rotation-interval   60000 ; milliseconds
 :credentials         {:access-key ""
                       :secret-key ""
                       :endpoint "s3-eu-west-1.amazonaws.com"}
 :bucket              "my-bifrost-bucket"
 :riemann-host        "localhost"}<commit_msg>Add comments to example configuration file.<commit_after>{:consumer-properties {"zookeeper.connect"  "localhost:2181"
                       "group.id"           "bifrost"
                       "auto.offset.reset"  "smallest" ; we explicitly commit offsets once files have
                                                       ; been uploaded to s3 so no need for auto commit
                       "auto.commit.enable" "false"}
 :topic-blacklist     #{"sometopic"} ; topics from :topic-blacklist will not be backed up
 :topic-whitelist     nil ; if :topic-whitelist is defined, only topics
                          ; from the whitelist will be backed up. The
                          ; value should be a set of strings.
 :rotation-interval   60000 ; milliseconds
 :credentials         {:access-key ""
                       :secret-key ""
                       :endpoint "s3-eu-west-1.amazonaws.com"}
 :bucket              "my-bifrost-bucket"
 :riemann-host        nil ; if :riemann-host is set, metrics will be pushed to that host
 }
{'fraction_non_alphanumeric': 0.09194097616345062, 'fraction_numerical': 0.01362088535754824, 'mean_word_length': 1.5366906474820143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '773819', 'n_tokens_mistral': 461, 'n_tokens_neox': 430, 'n_words': 128}