<commit_before>import functools
import gc
import gevent
import logging
import pymongo
from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure, TimeoutError
import signal
import sys
import re

loggers = {}
def get_logger(name):
    """
    get a logger object with reasonable defaults for formatting

    @param name used to identify the logger (though not currently useful for anything)
    """
    global loggers
    if name in loggers:
        return loggers[name]

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s:%(processName)s] %(message)s",
                                      "%m-%d %H:%M:%S"))
    logger.addHandler(sh)

    loggers[name] = logger
    return logger

log = get_logger("utils")


def mongo_connect(config, ensure_direct=False, secondary_only=False, max_pool_size=1,
                  socketTimeoutMS=None, w=0, read_preference=None, document_class=dict,
                  replicaSet=None, slave_okay=None):
    """
    Same as MongoClient.connect, except that we are paranoid and ensure that cursors
    # point to what we intended to point them to. Also sets reasonable defaults for our
    needs.

    @param host            host to connect to
    @param port            port to connect to
    @param ensure_direct   do safety checks to ensure we are connected to specified mongo instance

    most other keyword arguments mirror those for pymongo.MongoClient
    """
    options = dict(
        host=config['host'],
        port=config['port'],
        socketTimeoutMS=socketTimeoutMS,
        use_greenlets=True,
        max_pool_size=max_pool_size,
        w=1,
        document_class=document_class)
    if replicaSet is not None:
        options['replicaSet'] = replicaSet
    if read_preference is not None:
        options['read_preference'] = read_preference
    if slave_okay is not None:
        options['slave_okay'] = slave_okay
    client = pymongo.MongoClient(**options)

    if ensure_direct:
        # make sure we are connected to mongod/mongos that was specified; mongodb drivers
        # have the tendency of doing "magical" things in terms of connecting to other boxes
        test_collection = client['local']['test']
        test_cursor = test_collection.find(slave_okay=True, limit=1)
        connection = test_cursor.collection.database.connection
        if connection.host != host or connection.port != port:
            raise ValueError("connected to %s:%d (expected %s:%d)" %
                             (connection.host, connection.port, host, port))

    db = config['db']
    if not client[db].authenticate(config['user'], config['password']):
        raise ValueError("failed to authenticate to %s:%s with user %s and supplied password" % (host, port, user))

    return client


def parse_mongo_url(url):
    """
    Takes in pseudo-URL of form

    host[:port]/db/collection (e.g. localhost/prod_maestro/emails)

    and returns a dictionary containing elements 'host', 'port', 'db', 'collection'
    """
    try:
        parts = re.split('[/:@]', url)
        if len(parts) == 3
          host, db, collection = parts
        elif len(parts) == 5
          user, password, host, db, collection = parts
    except ValueError:
        raise ValueError("urls be of format: [user:password@]host[:port]/db/collection")

    host_tokens = host.split(':')
    if len(host_tokens) == 2:
        host = host_tokens[0]
        port = int(host_tokens[1])
    elif len(host_tokens) == 1:
        port = 27017
    elif len(host_tokens) > 2:
        raise ValueError("urls be of format: host[:port]/db/collection")

    return dict(host=host, port=port, db=db, collection=collection, user=user, password=password)


def _source_file_syntax():
    print "--source files must be of the following format:"
    print "database_name.collection_name"<commit_msg>stupid python mistake<commit_after>import functools
import gc
import gevent
import logging
import pymongo
from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure, TimeoutError
import signal
import sys
import re

loggers = {}
def get_logger(name):
    """
    get a logger object with reasonable defaults for formatting

    @param name used to identify the logger (though not currently useful for anything)
    """
    global loggers
    if name in loggers:
        return loggers[name]

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s:%(processName)s] %(message)s",
                                      "%m-%d %H:%M:%S"))
    logger.addHandler(sh)

    loggers[name] = logger
    return logger

log = get_logger("utils")


def mongo_connect(config, ensure_direct=False, secondary_only=False, max_pool_size=1,
                  socketTimeoutMS=None, w=0, read_preference=None, document_class=dict,
                  replicaSet=None, slave_okay=None):
    """
    Same as MongoClient.connect, except that we are paranoid and ensure that cursors
    # point to what we intended to point them to. Also sets reasonable defaults for our
    needs.

    @param host            host to connect to
    @param port            port to connect to
    @param ensure_direct   do safety checks to ensure we are connected to specified mongo instance

    most other keyword arguments mirror those for pymongo.MongoClient
    """
    options = dict(
        host=config['host'],
        port=config['port'],
        socketTimeoutMS=socketTimeoutMS,
        use_greenlets=True,
        max_pool_size=max_pool_size,
        w=1,
        document_class=document_class)
    if replicaSet is not None:
        options['replicaSet'] = replicaSet
    if read_preference is not None:
        options['read_preference'] = read_preference
    if slave_okay is not None:
        options['slave_okay'] = slave_okay
    client = pymongo.MongoClient(**options)

    if ensure_direct:
        # make sure we are connected to mongod/mongos that was specified; mongodb drivers
        # have the tendency of doing "magical" things in terms of connecting to other boxes
        test_collection = client['local']['test']
        test_cursor = test_collection.find(slave_okay=True, limit=1)
        connection = test_cursor.collection.database.connection
        if connection.host != host or connection.port != port:
            raise ValueError("connected to %s:%d (expected %s:%d)" %
                             (connection.host, connection.port, host, port))

    db = config['db']
    if not client[db].authenticate(config['user'], config['password']):
        raise ValueError("failed to authenticate to %s:%s with user %s and supplied password" % (host, port, user))

    return client


def parse_mongo_url(url):
    """
    Takes in pseudo-URL of form

    host[:port]/db/collection (e.g. localhost/prod_maestro/emails)

    and returns a dictionary containing elements 'host', 'port', 'db', 'collection'
    """
    try:
        parts = re.split('[/:@]', url)
        if len(parts) == 3:
          host, db, collection = parts
        elif len(parts) == 5:
          user, password, host, db, collection = parts
    except ValueError:
        raise ValueError("urls be of format: [user:password@]host[:port]/db/collection")

    host_tokens = host.split(':')
    if len(host_tokens) == 2:
        host = host_tokens[0]
        port = int(host_tokens[1])
    elif len(host_tokens) == 1:
        port = 27017
    elif len(host_tokens) > 2:
        raise ValueError("urls be of format: host[:port]/db/collection")

    return dict(host=host, port=port, db=db, collection=collection, user=user, password=password)


def _source_file_syntax():
    print "--source files must be of the following format:"
    print "database_name.collection_name"
{'fraction_non_alphanumeric': 0.09959555106167846, 'fraction_numerical': 0.004044489383215369, 'mean_word_length': 2.9904185577407967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2286852', 'n_tokens_mistral': 2282, 'n_tokens_neox': 2221, 'n_words': 687}