<commit_before>import json
import logging
import logging.config
import os
import sys

import twitter
from pymongo import MongoClient

import config
import constants

conf = None
twitter_api = None
mongo_client = None


def _setup_logging(
        path='logging.json',
        log_level=logging.DEBUG,
        env_key='LOG_CFG'
):
    env_path = os.getenv(env_key, None)
    if env_path:
        path = env_path
    if os.path.exists(path):
        with open(path, 'rt') as f:
            log_conf = json.load(f)
        logging.config.dictConfig(log_conf)
    else:
        logging.basicConfig(level=log_level)


def _setup_twitter():
    logging.info("Setting up Twitter API client...")

    global twitter_api
    twitter_api = twitter.Api(
        consumer_key=conf.config["twitter"]["consumer_key"],
        consumer_secret=conf.config["twitter"]["consumer_secret"],
        access_token_key=conf.config["twitter"]["access_token_key"],
        access_token_secret=conf.config["twitter"]["access_token_secret"])

    logging.info("Verifying Twitter API client credentials...")
    twitter_api.VerifyCredentials()
    logging.info("Twitter API client credentials verified.")

    logging.info("Twitter API client set up.")


def _setup_mongo():
    logging.info("Setting up MongoDB client...")

    global mongo_client
    mongo_client = MongoClient(conf.config["mongodb"]["uri"])

    logging.info("Verifying MongoDB client credentials...")
    db = mongo_client[conf.config["mongodb"]["database"]]
    db.authenticate(
        conf.config["mongodb"]["user"],
        conf.config["mongodb"]["password"],
        mechanism=conf.config["mongodb"]["mechanism"])
    logging.info("MongoDB client credentials verified.")

    logging.info("MongoDB client set up.")


def main():
    # configure logger
    _setup_logging()

    # read configuration
    global conf
    try:
        conf = config.Configuration(constants.CONFIG_FILE_NAME)
    except Exception as exc:
        logging.error("Error getting configuration.", exc)
        sys.exit(1)

    # setup Twitter API client
    try:
        _setup_twitter()
    except Exception as exc:
        logging.error("Error setting up Twitter API client.", exc)
        sys.exit(1)<commit_msg>Use global MongoDB client var and databse names.<commit_after>import json
import logging
import logging.config
import os
import sys

import twitter
from pymongo import MongoClient

import config
import constants

DB_EPHEMERIS = 'ephemeris'
DB_FILES = 'files'

conf = None
twitter_api = None
mongo_client = None
mongo_db = None


def _setup_logging(
        path='logging.json',
        log_level=logging.DEBUG,
        env_key='LOG_CFG'
):
    env_path = os.getenv(env_key, None)
    if env_path:
        path = env_path
    if os.path.exists(path):
        with open(path, 'rt') as f:
            log_conf = json.load(f)
        logging.config.dictConfig(log_conf)
    else:
        logging.basicConfig(level=log_level)


def _setup_twitter():
    logging.info("Setting up Twitter API client...")

    global twitter_api
    twitter_api = twitter.Api(
        consumer_key=conf.config["twitter"]["consumer_key"],
        consumer_secret=conf.config["twitter"]["consumer_secret"],
        access_token_key=conf.config["twitter"]["access_token_key"],
        access_token_secret=conf.config["twitter"]["access_token_secret"])

    logging.info("Verifying Twitter API client credentials...")
    twitter_api.VerifyCredentials()
    logging.info("Twitter API client credentials verified.")

    logging.info("Twitter API client set up.")


def _setup_mongo():
    logging.info("Setting up MongoDB client...")

    global mongo_client
    global mongo_db
    mongo_client = MongoClient(conf.config["mongodb"]["uri"])

    logging.info("Verifying MongoDB client credentials...")
    mongo_db = mongo_client[conf.config["mongodb"]["database"]]
    mongo_db.authenticate(
        conf.config["mongodb"]["user"],
        conf.config["mongodb"]["password"],
        mechanism=conf.config["mongodb"]["mechanism"])
    logging.info("MongoDB client credentials verified.")

    logging.info("MongoDB client set up.")


def main():
    # configure logger
    _setup_logging()

    # read configuration
    global conf
    try:
        conf = config.Configuration(constants.CONFIG_FILE_NAME)
    except Exception as exc:
        logging.error("Error getting configuration.", exc)
        sys.exit(1)

    # setup Twitter API client
    try:
        _setup_twitter()
    except Exception as exc:
        logging.error("Error setting up Twitter API client.", exc)
        sys.exit(1)
{'fraction_non_alphanumeric': 0.11468288444830582, 'fraction_numerical': 0.0008688097306689834, 'mean_word_length': 3.340245051837889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5075460', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1345, 'n_words': 310}