<commit_before>#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import os
import pickle

import json

import sacred.optional as opt
from sacred.serializer import flatten, restore

__all__ = ('load_config_file', 'save_config_file')


class Handler(object):
    def __init__(self, load, dump, mode):
        self.load = load
        self.dump = dump
        self.mode = mode


HANDLER_BY_EXT = {
    '.json': Handler(lambda fp: restore(json.load(fp)),
                     lambda obj, fp: json.dump(flatten(obj), fp,
                                               sort_keys=True, indent=2), ''),
    '.pickle': Handler(pickle.load, pickle.dump, 'b'),
}


if opt.has_yaml:
    HANDLER_BY_EXT['.yaml'] = Handler(opt.yaml.load, opt.yaml.dump, '')
    HANDLER_BY_EXT['.yml'] = Handler(opt.yaml.load, opt.yaml.dump, '')


def get_handler(filename):
    _, extension = os.path.splitext(filename)
    return HANDLER_BY_EXT[extension]


def load_config_file(filename):
    handler = get_handler(filename)
    with open(filename, 'r' + handler.mode) as f:
        return handler.load(f)


def save_config_file(config, filename):
    handler = get_handler(filename)
    with open(filename, 'w' + handler.mode) as f:
        handler.dump(config, f)<commit_msg>improve get_handler exception<commit_after>#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import os
import pickle

import json

import sacred.optional as opt
from sacred.serializer import flatten, restore

__all__ = ('load_config_file', 'save_config_file')


class Handler(object):
    def __init__(self, load, dump, mode):
        self.load = load
        self.dump = dump
        self.mode = mode


HANDLER_BY_EXT = {
    '.json': Handler(lambda fp: restore(json.load(fp)),
                     lambda obj, fp: json.dump(flatten(obj), fp,
                                               sort_keys=True, indent=2), ''),
    '.pickle': Handler(pickle.load, pickle.dump, 'b'),
}


if opt.has_yaml:
    HANDLER_BY_EXT['.yaml'] = Handler(opt.yaml.load, opt.yaml.dump, '')
    HANDLER_BY_EXT['.yml'] = Handler(opt.yaml.load, opt.yaml.dump, '')


def get_handler(filename):
    _, extension = os.path.splitext(filename)
    try:
        return HANDLER_BY_EXT[extension]
    except KeyError:
        raise ValueError(
            '"{}" has invalid or unsupported extention'.format(filename)
        )


def load_config_file(filename):
    handler = get_handler(filename)
    with open(filename, 'r' + handler.mode) as f:
        return handler.load(f)


def save_config_file(config, filename):
    handler = get_handler(filename)
    with open(filename, 'w' + handler.mode) as f:
        handler.dump(config, f)
{'fraction_non_alphanumeric': 0.12414042707202316, 'fraction_numerical': 0.0014477017734346724, 'mean_word_length': 3.0527859237536656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7294284', 'n_tokens_mistral': 914, 'n_tokens_neox': 874, 'n_words': 201}