<commit_before>import os

import logging
logger = logging.getLogger('config')

import bson.json_util as json

DEFAULT_FILENAME = 'compdb.rc'
CONFIG_FILENAMES = ['compdb.rc',]
HOME = os.path.expanduser('~')
CONFIG_PATH = [HOME]
CWD = os.getcwd()

ENVIRONMENT_VARIABLES = {
    'author_name' :         'COMPDB_AUTHOR_NAME',
    'author_email':         'COMPDB_AUTHOR_EMAIL',
    'project':              'COMPDB_PROJECT',
    'project_dir' :         'COMPDB_PROJECT_DIR',
    'filestorage_dir':      'COMPDB_FILESTORAGE_DIR',
    'workspace_dir':        'COMPDB_WORKING_DIR',
    'database_host':        'COMPDB_DATABASE_HOST',
    'develop':              'COMPDB_DEVELOP',
    'connect_timeout_ms':   'COMPDB_CONNECT_TIMEOUT',
    'compmatdb_host':       'COMPDB_COMPMATDB_HOST',
}

REQUIRED_KEYS = [
    'author_name', 'author_email', 'project',
    'project_dir',  'filestorage_dir', 'workspace_dir',
    ]

DEFAULTS = {
    'database_host': 'localhost',
    'database_meta': 'compdb',
    'database_global_fs': 'compdb_fs',
    'database_compmatdb': 'compmatdb',
}

LEGAL_ARGS = REQUIRED_KEYS + list(DEFAULTS.keys()) + [
    'global_fs_dir', 'develop', 'connect_timeout_ms', 'compmatdb_host',
    ]

DIRS = ['workspace_dir', 'project_dir', 'filestorage_dir', 'global_fs_dir']

class Config(object):   

    def __init__(self, args = None):
        from copy import copy
        self._args = {}
        if args is not None:
            self.update(args)

    def __str__(self):
        return str(self._args)

    def read(self, filename = DEFAULT_FILENAME):
        with open(filename) as file:
            args = json.loads(file.read())
            logger.debug("Read: {}".format(args))
        self._args.update(args)

    def _read_files(self):
        from os.path import dirname
        root = None
        for fn in search_config_files():
            try:
                logger.debug("Reading config file '{}'.".format(fn))
                self.read(fn)
                if root is None and 'project' in self:
                    root = dirname(fn)
            except Exception as error:
                msg = "Error while reading config file '{}'."
                logger.error(msg.format(fn))
        if root is not None:
            logger.debug("Found root: {}".format(dirname(fn)))
            self['project_dir'] = root

    def update(self, args):
        self._args.update(args)

    def load(self):
        logger.debug('Reading config...')
        self._read_files()
        self._args.update(read_environment())
        logger.debug('Verifying config...')
        self.verify()
        logger.debug('OK')

    def verify(self):
        verify(self._args)

    def write(self, filename = DEFAULT_FILENAME, indent = 0, keys = None):
        if keys is None:
            args = self._args
        else:
            args = {k: self._args[k] for k in keys if k in self._args}
        with open(filename, 'w') as file:
            json.dump(args, file, indent = indent)

    def _dump(self, indent = 0, keys = None):
        if keys is None:
            args = self._args
        else:
            args = {k: self._args[k] for k in keys if k in self._args}
        return json.dumps(args, indent = indent, sort_keys = True)

    def dump(self, indent = 0, keys = None):
        print(self._dump(indent, keys))

    def __str__(self):
        return self._dump(indent = 1)

    def __getitem__(self, key):
        try:
            v = self._args.get(key)
            return DEFAULTS[key] if v is None else v
        except KeyError as error:
            msg = "Missing config key: '{key}'. "
            msg += "Try 'compdb config add {key} [your_value]"
            raise KeyError(msg.format(key = key)) from error

    def get(self, key, default = None):
        return self._args.get(key, default)
<commit_msg>Fixed config module, use basic json package.<commit_after>import os

import logging
logger = logging.getLogger('config')

import json

DEFAULT_FILENAME = 'compdb.rc'
CONFIG_FILENAMES = ['compdb.rc',]
HOME = os.path.expanduser('~')
CONFIG_PATH = [HOME]
CWD = os.getcwd()

ENVIRONMENT_VARIABLES = {
    'author_name' :         'COMPDB_AUTHOR_NAME',
    'author_email':         'COMPDB_AUTHOR_EMAIL',
    'project':              'COMPDB_PROJECT',
    'project_dir' :         'COMPDB_PROJECT_DIR',
    'filestorage_dir':      'COMPDB_FILESTORAGE_DIR',
    'workspace_dir':        'COMPDB_WORKING_DIR',
    'database_host':        'COMPDB_DATABASE_HOST',
    'develop':              'COMPDB_DEVELOP',
    'connect_timeout_ms':   'COMPDB_CONNECT_TIMEOUT',
    'compmatdb_host':       'COMPDB_COMPMATDB_HOST',
}

REQUIRED_KEYS = [
    'author_name', 'author_email', 'project',
    'project_dir',  'filestorage_dir', 'workspace_dir',
    ]

DEFAULTS = {
    'database_host': 'localhost',
    'database_meta': 'compdb',
    'database_global_fs': 'compdb_fs',
    'database_compmatdb': 'compmatdb',
}

LEGAL_ARGS = REQUIRED_KEYS + list(DEFAULTS.keys()) + [
    'global_fs_dir', 'develop', 'connect_timeout_ms', 'compmatdb_host',
    ]

DIRS = ['workspace_dir', 'project_dir', 'filestorage_dir', 'global_fs_dir']

class Config(object):   

    def __init__(self, args = None):
        from copy import copy
        self._args = {}
        if args is not None:
            self.update(args)

    def __str__(self):
        return str(self._args)

    def read(self, filename = DEFAULT_FILENAME):
        try:
            with open(filename) as file:
                args = json.loads(file.read())
                logger.debug("Read: {}".format(args))
            self._args.update(args)
        except ValueError as error:
            msg = "Failed to read config file '{}'."
            raise RuntimeError(msg.format(filename))

    def _read_files(self):
        from os.path import dirname
        root = None
        for fn in search_config_files():
            try:
                logger.debug("Reading config file '{}'.".format(fn))
                self.read(fn)
                if root is None and 'project' in self:
                    root = dirname(fn)
            except Exception as error:
                msg = "Error while reading config file '{}'."
                logger.error(msg.format(fn))
        if root is not None:
            logger.debug("Found root: {}".format(dirname(fn)))
            self['project_dir'] = root

    def update(self, args):
        self._args.update(args)

    def load(self):
        logger.debug('Reading config...')
        self._read_files()
        self._args.update(read_environment())
        logger.debug('Verifying config...')
        self.verify()
        logger.debug('OK')

    def verify(self):
        verify(self._args)

    def write(self, filename = DEFAULT_FILENAME, indent = 0, keys = None):
        import tempfile
        if keys is None:
            args = self._args
        else:
            args = {k: self._args[k] for k in keys if k in self._args}
        with tempfile.NamedTemporaryFile() as file:
            with open(filename, 'w') as file:
                json.dump(args, file, indent = indent)
            os.rename(file.name, filename)

    def _dump(self, indent = 0, keys = None):
        if keys is None:
            args = self._args
        else:
            args = {k: self._args[k] for k in keys if k in self._args}
        return json.dumps(args, indent = indent, sort_keys = True)

    def dump(self, indent = 0, keys = None):
        print(self._dump(indent, keys))

    def __str__(self):
        return self._dump(indent = 1)

    def __getitem__(self, key):
        try:
            v = self._args.get(key)
            return DEFAULTS[key] if v is None else v
        except KeyError as error:
            msg = "Missing config key: '{key}'. "
            msg += "Try 'compdb config add {key} [your_value]"
            raise KeyError(msg.format(key = key)) from error

    def get(self, key, default = None):
        return self._args.get(key, default)

{'fraction_non_alphanumeric': 0.12300540268877999, 'fraction_numerical': 0.001005151400929765, 'mean_word_length': 2.22920892494929, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4089346', 'n_tokens_mistral': 2514, 'n_tokens_neox': 2489, 'n_words': 605}