<commit_before>#!/usr/bin/python
# coding=utf-8

from __future__ import division, print_function, unicode_literals

import argparse
import collections
import json


def recursive_update(d, u):
    for k, v in u.iteritems():
        if isinstance(v, collections.Mapping):
            r = recursive_update(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d


def parse_arguments():
    parser = argparse.ArgumentParser()

    parser.add_argument("update", nargs='*',
                        help='updates to the default options of the form '
                             'foo.bar=baz')

    parser.add_argument("-O", "--config_file",
                        help="JSON file to overwrite default configuration")

    parser.add_argument("-M", "--mongo_url", nargs='?',
                        help='activate the mongoDB obeserver and optionally '
                             'pass an url',
                        const="localhost:27017")

    parser.add_argument("-m", "--mongo_database",
                        help='name of the MongoDB database to use',
                        default='sperment')

    args = parser.parse_args()

    config_updates = {}

    if args.config_file:
        with open(args.config_file, 'r') as f:
            config_updates = json.load(f)

    if args.update:
        for upd in args.update:
            split_update = upd.split('=')
            assert len(split_update) == 2
            path, value = split_update
            current_option = config_updates
            for p in path.split('.')[:-1]:
                if p not in current_option:
                    current_option[p] = dict()
                current_option = current_option[p]
            try:
                converted_value = json.loads(value)
            except ValueError:
                converted_value = value

            current_option[path.split('.')[-1]] = converted_value

    observers = []

    if args.mongo_url:
        from mlite.observers.mongodb import MongoDBReporter
        mongo = MongoDBReporter(db_name=args.mongo_database,
                                url=args.mongo_url)
        observers.append(mongo)

    return config_updates, observers<commit_msg>fixed mongodb issue<commit_after>#!/usr/bin/python
# coding=utf-8

from __future__ import division, print_function, unicode_literals

import argparse
import collections
import json
from sperment.observers import MongoDBReporter


def recursive_update(d, u):
    for k, v in u.iteritems():
        if isinstance(v, collections.Mapping):
            r = recursive_update(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d


def parse_arguments():
    parser = argparse.ArgumentParser()

    parser.add_argument("update", nargs='*',
                        help='updates to the default options of the form '
                             'foo.bar=baz')

    parser.add_argument("-O", "--config_file",
                        help="JSON file to overwrite default configuration")

    parser.add_argument("-M", "--mongo_url", nargs='?',
                        help='activate the mongoDB obeserver and optionally '
                             'pass an url',
                        const="localhost:27017")

    parser.add_argument("-m", "--mongo_database",
                        help='name of the MongoDB database to use',
                        default='sperment')

    args = parser.parse_args()

    config_updates = {}

    if args.config_file:
        with open(args.config_file, 'r') as f:
            config_updates = json.load(f)

    if args.update:
        for upd in args.update:
            split_update = upd.split('=')
            assert len(split_update) == 2
            path, value = split_update
            current_option = config_updates
            for p in path.split('.')[:-1]:
                if p not in current_option:
                    current_option[p] = dict()
                current_option = current_option[p]
            try:
                converted_value = json.loads(value)
            except ValueError:
                converted_value = value

            current_option[path.split('.')[-1]] = converted_value

    observers = []

    if args.mongo_url:
        mongo = MongoDBReporter(db_name=args.mongo_database,
                                url=args.mongo_url)
        observers.append(mongo)

    return config_updates, observers
{'fraction_non_alphanumeric': 0.10263929618768329, 'fraction_numerical': 0.004060455673358899, 'mean_word_length': 1.774718397997497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4001256', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1207, 'n_words': 291}