<commit_before>from __future__ import division
"""
Author: <NAME>
"""
__license__ = """
Copyright 2015 Parse.ly, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__all__ = ["Cluster"]
import logging
import random
import time
import weakref

from .broker import Broker
from .exceptions import (ConsumerCoordinatorNotAvailable,
                         UnknownTopicOrPartition)
from .protocol import ConsumerMetadataRequest, ConsumerMetadataResponse
from .topic import Topic


log = logging.getLogger(__name__)


class TopicDict(dict):
    """Dictionary which will attempt to auto-create unknown topics."""

    def __init__(self, cluster, *args, **kwargs):
        super(TopicDict, self).__init__(*args, **kwargs)
        self._cluster = weakref.proxy(cluster)

    def __missing__(self, key):
        log.warning('Topic %s not found. Attempting to auto-create.', key)
        if self._create_topic(key):
            return self[key]
        else:
            raise UnknownTopicOrPartition('Unknown topic: {}'.format(key))

    def _create_topic(self, topic_name):
        """Auto-create a topic.

        Not exposed in the cluster or broker because this is *only*
        auto-creation.  When there's a real API for creating topics,
        with settings and everything, we'll implement that. To expose just
        this now would be disingenuous, since it's features would be hobbled.
        """
        # Auto-creating will take a moment, so we try 5 times.
        for i in xrange(5):
            # Auto-creating is as simple as issuing a metadata request
            # solely for that topic.  The update is just to be sure
            # our `Cluster` knows about it.
            self._cluster.brokers[self._cluster.brokers.keys()[0]].request_metadata(topics=[topic_name])
            self._cluster.update()
            if topic_name in self:
                log.info('Topic %s successfully auto-created.', topic_name)
                return True
            time.sleep(0.1)

<commit_msg>Add handling for bad metadata until it's fixed in Kafka. [Fixes #175]<commit_after>from __future__ import division
"""
Author: <NAME>
"""
__license__ = """
Copyright 2015 Parse.ly, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__all__ = ["Cluster"]
import logging
import random
import time
import weakref

from .broker import Broker
from .exceptions import (ConsumerCoordinatorNotAvailable,
                         KafkaException,
                         UnknownTopicOrPartition)
from .protocol import ConsumerMetadataRequest, ConsumerMetadataResponse
from .topic import Topic


log = logging.getLogger(__name__)


class TopicDict(dict):
    """Dictionary which will attempt to auto-create unknown topics."""

    def __init__(self, cluster, *args, **kwargs):
        super(TopicDict, self).__init__(*args, **kwargs)
        self._cluster = weakref.proxy(cluster)

    def __missing__(self, key):
        log.warning('Topic %s not found. Attempting to auto-create.', key)
        if self._create_topic(key):
            return self[key]
        else:
            raise UnknownTopicOrPartition('Unknown topic: {}'.format(key))

    def _create_topic(self, topic_name):
        """Auto-create a topic.

        Not exposed in the cluster or broker because this is *only*
        auto-creation.  When there's a real API for creating topics,
        with settings and everything, we'll implement that. To expose just
        this now would be disingenuous, since it's features would be hobbled.
        """
        if len(self._cluster.brokers) == 0:
            log.warning("No brokers found. This is probably because of "
                        "KAFKA-2154, which will be fixed in Kafka 0.8.3")
            raise KafkaException("No brokers configured. Can't auto-create topic.")
        # Auto-creating will take a moment, so we try 5 times.
        for i in xrange(5):
            # Auto-creating is as simple as issuing a metadata request
            # solely for that topic.  The update is just to be sure
            # our `Cluster` knows about it.
            self._cluster.brokers[self._cluster.brokers.keys()[0]].request_metadata(topics=[topic_name])
            self._cluster.update()
            if topic_name in self:
                log.info('Topic %s successfully auto-created.', topic_name)
                return True
            time.sleep(0.1)


{'fraction_non_alphanumeric': 0.08239489489489489, 'fraction_numerical': 0.006944444444444444, 'mean_word_length': 2.825556353194544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7086007', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1348, 'n_words': 556}