<commit_before>"""
Utility functions used across scripts
"""
__author__ = "<NAME>"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "1.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__date__ = "Dec 1, 2012"

import bson
import datetime
import json
import os
from pymongo import Connection


DEFAULT_PORT = 27017

DEFAULT_SETTINGS = [
    ("host", "localhost"),
    ("port", DEFAULT_PORT),
    ("database", "vasp"),
    ("admin_user", None),
    ("admin_password", None),
    ("readonly_user", None),
    ("readonly_password", None),
    ("collection", "tasks"),
    ("aliases_config", None),
    ("mapi_key", None)
]


def get_settings(config_file):
    if config_file:
        with open(config_file) as f:
            return json.load(f)
    elif os.path.exists("db.json"):
        with open("db.json") as f:
            return json.load(f)
    else:
        return dict(DEFAULT_SETTINGS)


def get_collection(config_file, admin=False):
    d = get_settings(config_file)
    conn = Connection(d["host"], d["port"])
    db = conn[d["database"]]
    user = d["admin_user"] if admin else d["readonly_user"]
    passwd = d["admin_password"] if admin else d["readonly_password"]
    db.authenticate(user, passwd)
    return db[d["collection"]]


class MongoJSONEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, bson.objectid.ObjectId):
            return str(o)
        if isinstance(o, datetime.datetime):
            return o.isoformat()
        return json.JSONEncoder.default(self, o)<commit_msg>Add a get_database utility function.<commit_after>"""
Utility functions used across scripts
"""
__author__ = "<NAME>"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "1.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__date__ = "Dec 1, 2012"

import bson
import datetime
import json
import os
from pymongo import Connection


DEFAULT_PORT = 27017

DEFAULT_SETTINGS = [
    ("host", "localhost"),
    ("port", DEFAULT_PORT),
    ("database", "vasp"),
    ("admin_user", None),
    ("admin_password", None),
    ("readonly_user", None),
    ("readonly_password", None),
    ("collection", "tasks"),
    ("aliases_config", None),
    ("mapi_key", None)
]


def get_settings(config_file):
    if config_file:
        with open(config_file) as f:
            return json.load(f)
    elif os.path.exists("db.json"):
        with open("db.json") as f:
            return json.load(f)
    else:
        return dict(DEFAULT_SETTINGS)


def get_database(config_file, admin=False):
    d = get_settings(config_file)
    conn = Connection(d["host"], d["port"])
    db = conn[d["database"]]
    user = d["admin_user"] if admin else d["readonly_user"]
    passwd = d["admin_password"] if admin else d["readonly_password"]
    db.authenticate(user, passwd)
    return db


def get_collection(config_file, admin=False):
    db = get_database(config_file, admin=admin)
    return db[d["collection"]]


class MongoJSONEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, bson.objectid.ObjectId):
            return str(o)
        if isinstance(o, datetime.datetime):
            return o.isoformat()
        return json.JSONEncoder.default(self, o)
{'fraction_non_alphanumeric': 0.13905233818519666, 'fraction_numerical': 0.009910188912976153, 'mean_word_length': 3.5301542776998596, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3168922', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1014, 'n_words': 259}