<commit_before>import itertools
from collections import defaultdict
from Queue import Queue

from kafka import base
from kafka.common import OffsetType

# Settings to add to the eventual BalancedConsumer
# consumer_group
# rebalance_max_retries
# rebalance_backoff_ms
# partition_assignment_strategy


class SimpleConsumer(base.BaseSimpleConsumer):

    def __init__(self,
                 topic,
                 client_id=None,
                 consumer_group=None,
                 partitions=None,
                 socket_timout_ms=30000,
                 socket_receive_buffer_bytes=60*1024,
                 fetch_message_max_bytes=1024*1024,
                 num_consumer_fetchers=1,
                 auto_commit_enable=False,
                 auto_commit_interval_ms=60*1000,
                 queued_max_message_chunks=2,
                 fetch_min_bytes=1,
                 fetch_wait_max_ms=100,
                 refresh_leader_backoff_ms=200,
                 offsets_channel_backoff_ms=1000,<commit_msg>PEP8 SimpleConsumer<commit_after>import itertools
from collections import defaultdict
from Queue import Queue

from kafka import base
from kafka.common import OffsetType

# Settings to add to the eventual BalancedConsumer
# consumer_group
# rebalance_max_retries
# rebalance_backoff_ms
# partition_assignment_strategy


class SimpleConsumer(base.BaseSimpleConsumer):

    def __init__(self,
                 topic,
                 client_id=None,
                 consumer_group=None,
                 partitions=None,
                 socket_timout_ms=30000,
                 socket_receive_buffer_bytes=60 * 1024,
                 fetch_message_max_bytes=1024 * 1024,
                 num_consumer_fetchers=1,
                 auto_commit_enable=False,
                 auto_commit_interval_ms=60 * 1000,
                 queued_max_message_chunks=2,
                 fetch_min_bytes=1,
                 fetch_wait_max_ms=100,
                 refresh_leader_backoff_ms=200,
                 offsets_channel_backoff_ms=1000,
{'fraction_non_alphanumeric': 0.04603330068560235, 'fraction_numerical': 0.03770812928501469, 'mean_word_length': 2.172360248447205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611630', 'n_tokens_mistral': 608, 'n_tokens_neox': 548, 'n_words': 99}