<commit_before>import functools
import gc
import gevent
import logging
import pymongo
from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure
from pymongo.uri_parser import parse_uri
import signal
import gevent
import sys
import re

gevent.monkey.patch_all()

loggers = {}
def get_logger(name):
    """
    get a logger object with reasonable defaults for formatting

    @param name used to identify the logger (though not currently useful for anything)
    """
    global loggers
    if name in loggers:
        return loggers[name]

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s:%(processName)s] %(message)s",
                                      "%m-%d %H:%M:%S"))
    logger.addHandler(sh)

    loggers[name] = logger
    return logger

log = get_logger("utils")


def mongo_connect(config, ensure_direct=False, secondary_only=False, max_pool_size=1,
                  socketTimeoutMS=None, w=0, read_preference=None, document_class=dict,
                  replicaSet=None):
    """
    Same as MongoClient.connect, except that we are paranoid and ensure that cursors
    # point to what we intended to point them to. Also sets reasonable defaults for our
    needs.

    @param host            host to connect to
    @param port            port to connect to
    @param ensure_direct   do safety checks to ensure we are connected to specified mongo instance

    most other keyword arguments mirror those for pymongo.MongoClient
    """
    
    options = dict(
        host=config['host'],
        port=config['port'],
        socketTimeoutMS=socketTimeoutMS,
        maxPoolSize=max_pool_size,
        w=1,
        document_class=document_class,
    )
    
    if read_preference is not None:
        options['read_preference'] = read_preference
    if replicaSet is not None:
        options['replicaSet'] = replicaSet
    if socketTimeoutMS is not None:
        options['socketTimeoutMS'] = socketTimeoutMS
    
    client = pymongo.MongoClient(**options)

    db = config['db']
    if not client[db].authenticate(
        config['user'],
        config['password'],
        source=config['options'].get('authsource',None)<commit_msg>Skip authentication if users not set in url<commit_after>import functools
import gc
import gevent
import logging
import pymongo
from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure
from pymongo.uri_parser import parse_uri
import signal
import gevent
import sys
import re

gevent.monkey.patch_all()

loggers = {}
def get_logger(name):
    """
    get a logger object with reasonable defaults for formatting

    @param name used to identify the logger (though not currently useful for anything)
    """
    global loggers
    if name in loggers:
        return loggers[name]

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s:%(processName)s] %(message)s",
                                      "%m-%d %H:%M:%S"))
    logger.addHandler(sh)

    loggers[name] = logger
    return logger

log = get_logger("utils")


def mongo_connect(config, ensure_direct=False, secondary_only=False, max_pool_size=1,
                  socketTimeoutMS=None, w=0, read_preference=None, document_class=dict,
                  replicaSet=None):
    """
    Same as MongoClient.connect, except that we are paranoid and ensure that cursors
    # point to what we intended to point them to. Also sets reasonable defaults for our
    needs.

    @param host            host to connect to
    @param port            port to connect to
    @param ensure_direct   do safety checks to ensure we are connected to specified mongo instance

    most other keyword arguments mirror those for pymongo.MongoClient
    """
    
    options = dict(
        host=config['host'],
        port=config['port'],
        socketTimeoutMS=socketTimeoutMS,
        maxPoolSize=max_pool_size,
        w=1,
        document_class=document_class,
    )
    
    if read_preference is not None:
        options['read_preference'] = read_preference
    if replicaSet is not None:
        options['replicaSet'] = replicaSet
    if socketTimeoutMS is not None:
        options['socketTimeoutMS'] = socketTimeoutMS
    
    client = pymongo.MongoClient(**options)

    db = config['db']
    if 'user' not in config:
        return client

    if not client[db].authenticate(
        config['user'],
        config['password'],
        source=config['options'].get('authsource',None)
{'fraction_non_alphanumeric': 0.08508358336905272, 'fraction_numerical': 0.0012858979854264896, 'mean_word_length': 2.9685374149659864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1126855', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1286, 'n_words': 393}