<commit_before>import logging
import pymongo
try:
    from monary import Monary
    MONARY_ENABLED = True
except ImportError:
    MONARY_ENABLED = False


class MongoManager:
    """Create clients for several databases, and shut them down properly
    """

    def __init__(self, config):
        self.config = config
        self.clients = {}
        self.log = logging.getLogger('MongoManager')
        self.monary_enabled = MONARY_ENABLED

    def get_database(self, database_name=None, uri=None, monary=False):
        """Get a connection to the database_name. Returns Mongo database access object.
        If you provide a mongodb connection string uri, it will be used, otherwise one will be built
        from the configuration settings.
        If database_name=None, will connect to the default database of the uri.
        If there is already a client connecting to this database, no new mongoclient will be created.
        """
        if uri is None:
            uri = 'mongodb://{user}:{password}@{host}:{port}/{database}'.format(database=database_name,
                                                                                **self.config)
        if monary:
            # Monary clients are not cached
            self.log.debug("Connecting to Mongo via monary using uri %s" % uri)
            return Monary(uri)

        elif database_name not in self.clients:
            self.log.debug("Connecting to Mongo using uri %s" % uri)
            client = pymongo.MongoClient(uri)

            if database_name is None:
                self.log.debug("Grabbing default database")
                database = client.get_default_database()
            else:
                self.log.debug("Grabbing database %s" % database_name)
                database = client[database_name]
            client.admin.command('ping')        # raises pymongo.errors.ConnectionFailure on failure
            self.log.debug("Succesfully pinged database %s" % database_name)

            self.clients[database_name] = dict(client=client, db=database)

        return self.clients[database_name]['db']

    def shutdown(self):
        self.log.debug('Shutting down all MongoDB clients')
        for v in self.clients.values():
            v['client'].close()
<commit_msg>Add user and password to MongoDB URI's<commit_after>import logging
import pymongo
import re
try:
    from monary import Monary
    MONARY_ENABLED = True
except ImportError:
    MONARY_ENABLED = False


class MongoManager:
    """Create clients for several databases, and shut them down properly
    """

    def __init__(self, config):
        self.config = config
        self.clients = {}
        self.log = logging.getLogger('MongoManager')
        self.monary_enabled = MONARY_ENABLED

    def get_database(self, database_name=None, uri=None, monary=False):
        """Get a connection to the database_name. Returns Mongo database access object.
        If you provide a mongodb connection string uri, we will insert user & password into it,
        otherwise one will be built from the configuration settings.
        If database_name=None, will connect to the default database of the uri.
        If there is already a client connecting to this database, no new mongoclient will be created.
        """
        # Pattern of URI's we expect from database (without user & pass)
        uri_pattern = r'mongodb://([^:]+):(\d+)/(\w+)'

        # Format of URI we should eventually send to mongo
        full_uri_format = 'mongodb://{user}:{password}@{host}:{port}/{database}'

        if uri is None:
            # Construct the entire URI from default settings
            uri = full_uri_format.format(database=database_name, **self.config)
        else:
            m = re.match(uri_pattern, uri)
            if m:
                # URI was provided, but without user & pass.
                host, port, database_name = m.groups()
                uri = full_uri_format.format(database=database_name, host=host, port=port,
                                             user=self.config['user'], password=self.config['password'])
            else:
                # Some other URI was provided. Maybe works...
                self.log.warning("Unexpected Mongo URI %s, expected format %s. Trying anyway..." % (uri, uri_pattern))

        if monary:
            # Monary clients are not cached
            self.log.debug("Connecting to Mongo via monary using uri %s" % uri)
            return Monary(uri)

        elif database_name not in self.clients:
            self.log.debug("Connecting to Mongo using uri %s" % uri)
            client = pymongo.MongoClient(uri)

            if database_name is None:
                self.log.debug("Grabbing default database")
                database = client.get_default_database()
            else:
                self.log.debug("Grabbing database %s" % database_name)
                database = client[database_name]
            client.admin.command('ping')        # raises pymongo.errors.ConnectionFailure on failure
            self.log.debug("Succesfully pinged database %s" % database_name)

            self.clients[database_name] = dict(client=client, db=database)

        return self.clients[database_name]['db']

    def shutdown(self):
        self.log.debug('Shutting down all MongoDB clients')
        for v in self.clients.values():
            v['client'].close()

{'fraction_non_alphanumeric': 0.08866629567798182, 'fraction_numerical': 0.0, 'mean_word_length': 2.2345530893821235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442361', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1378, 'n_words': 449}