<commit_before>#!/usr/bin/env python
"""
Get the data from EMDR and shove it into the redis queue
<NAME> <EMAIL>
"""

import zlib
import zmq.green as zmq
import gevent
from gevent.pool import Pool
from gevent import monkey; gevent.monkey.patch_all()
from hotqueue import HotQueue

# Max number of greenlet workers
MAX_NUM_POOL_WORKERS = 200

# DEBUG flag
DEBUG = False<commit_msg>Stupid import.  Added ConfigParser and the config file<commit_after>#!/usr/bin/env python
"""
Get the data from EMDR and shove it into the redis queue
<NAME> <EMAIL>
"""

import zlib
import zmq.green as zmq
import gevent
import ConfigParser
from gevent.pool import Pool
from gevent import monkey; gevent.monkey.patch_all()
from hotqueue import HotQueue

# Load connection params from the configuration file
config = ConfigParser.ConfigParser()
config.read('consumer.conf')
redisdb = config.get('Redis', 'redishost')


# Max number of greenlet workers
MAX_NUM_POOL_WORKERS = 200

# DEBUG flag
DEBUG = False
{'fraction_non_alphanumeric': 0.08020304568527918, 'fraction_numerical': 0.006091370558375634, 'mean_word_length': 5.1625, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481169', 'n_tokens_mistral': 322, 'n_tokens_neox': 308, 'n_words': 117}