<commit_before>from redis import Redis
import urlparse

class FeederQueue(object):

    def __init__(self, app):
        self.logger = app.logger # bit quick and dirty
        url = urlparse.urlparse(app.config.get('REDIS_HOST'))
        self.redis_queue = app.config.get('REDIS_QUEUE_KEY')
        self.redis = Redis(host=url.hostname, port=url.port, password=url.password)

    def enqueue(self, number, json):
        try:
            self.redis.rpush(self.redis_queue, json)
            self.logger .info("Enqueued data for title %s: data: %s" % (number, json))
            self.logger.info("Redis: %s" % self.redis)
        except Exception, e:
            self.logger .info(e)
            self.logger .info("Couldn't enqueue data for title %s: data: %s" % (number, json))<commit_msg>remove unicode characters<commit_after>from redis import Redis
import urlparse

class FeederQueue(object):

    def __init__(self, app):
        self.logger = app.logger # bit quick and dirty
        url = urlparse.urlparse(app.config.get('REDIS_HOST'))
        self.redis_queue = app.config.get('REDIS_QUEUE_KEY')
        self.redis = Redis(host=url.hostname, port=url.port, password=url.password)

    def enqueue(self, number, json):
        try:
            json_ascii = convert_to_ascii(json)
            self.redis.rpush(self.redis_queue, json_ascii)
            self.logger .info("Enqueued data for title %s: data: %s" % (number, json))
            self.logger.info("Redis: %s" % self.redis)
        except Exception, e:
            self.logger .info(e)
            self.logger .info("Couldn't enqueue data for title %s: data: %s" % (number, json))


def convert_to_ascii(input):
    if isinstance(input, dict):
        return {convert_to_ascii(key): convert_to_ascii(value) for key, value in input.iteritems()}
    elif isinstance(input, list):
        return [convert_to_ascii(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('utf-8')
    else:
        return input
{'fraction_non_alphanumeric': 0.12039800995024875, 'fraction_numerical': 0.0004975124378109452, 'mean_word_length': 2.8451242829827916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7427660', 'n_tokens_mistral': 645, 'n_tokens_neox': 631, 'n_words': 155}