<commit_before>import os
import collections
#
import crosscat.utils.file_utils as file_utils
import crosscat.utils.geweke_utils as geweke_utils
import crosscat.utils.general_utils as general_utils


result_filename = geweke_utils.summary_filename


def find_configs(dirname, filename=result_filename):
    root_has_filename = lambda (root, ds, filenames): filenames.count(filename)
    get_filepath = lambda (root, ds, fs): os.path.join(root, filename)
    tuples = filter(root_has_filename, os.walk(dirname))
    filepaths = map(get_filepath, tuples)
    return filepaths

def read_all_configs(dirname='.'):
    def read_config(filepath):
        result = file_utils.unpickle(filepath, dir=dirname)
        config = result['config']
        return config
    filepaths = find_configs(dirname)
    config_list = map(read_config, filepaths)
    return config_list

def generate_filepath(config):
    _dirname = geweke_utils.generate_directory_name(config)
    filepath = os.path.join(_dirname, result_filename)
    return filepath

def read_results(config_list, dirname=''):
    def read_result(config):
        filepath = generate_filepath(config)
        result = file_utils.unpickle(filepath, dir=dirname)
        return result
    config_list = general_utils.ensure_listlike(config_list)
    results = map(read_result, config_list)
    return results

def write_result(config, result, dirname=''):
    filepath = generate_filepath(config)
    file_utils.pickle(result, filepath, dirname=dirname)
    return

def do_experiments(runner, writer, config_list):
    def do_experiment(config):
        result = runner(**config)
        writer(config, result)
        return result
    config_list = general_utils.ensure_listlike(config_list)
    results = map(do_experiment, config_list)
    return results

if __name__ == '__main__':
    config_list = read_all_configs()
    results = read_results(config_list[:-1])
<commit_msg>refactor<commit_after>import os
import collections
#
import crosscat.utils.file_utils as file_utils
import crosscat.utils.geweke_utils as geweke_utils
import crosscat.utils.general_utils as general_utils


result_filename = geweke_utils.summary_filename
generate_directory_name = geweke_utils.generate_directory_name
writer = geweke_utils.write_result
reader = geweke_utils.read_result
runner = geweke_utils.run_geweke


def find_configs(dirname, filename=result_filename):
    root_has_filename = lambda (root, ds, filenames): filenames.count(filename)
    get_filepath = lambda (root, ds, fs): os.path.join(root, filename)
    def is_one_deep(filepath):
        _dir, _file = os.path.split(filepath)
        return os.path.split(_dir)[0] == dirname
    tuples = filter(root_has_filename, os.walk(dirname))
    filepaths = map(get_filepath, tuples)
    filepaths = filter(is_one_deep, filepaths)
    return filepaths

def read_all_configs(dirname='.'):
    def read_config(filepath):
        result = file_utils.unpickle(filepath, dir=dirname)
        config = result['config']
        return config
    filepaths = find_configs(dirname)
    config_list = map(read_config, filepaths)
    return config_list

def read_results(config_list, dirname=''):
    _read_result = lambda config: reader(config, dirname)
    config_list = general_utils.ensure_listlike(config_list)
    results = map(_read_result, config_list)
    return results

def write_results(results, dirname=''):
    _write_result = lambda result: writer(result, dirname)
    map(_write_result, results)
    return

def do_experiments(runner, writer, config_list, dirname=''):
    def do_experiment(config):
        result = runner(config)
        writer(result, dirname)
        return
    config_list = general_utils.ensure_listlike(config_list)
    map(do_experiment, config_list)
    return

def args_to_config(args):
    parser = geweke_utils.generate_parser()
    args = parser.parse_args(args)
    args = geweke_utils.arbitrate_args(args)
    return args.__dict__

if __name__ == '__main__':
    args_list = [
            ['--num_rows', '10', '--num_cols', '2', '--num_iters', '300', ],
            ['--num_rows', '10', '--num_cols', '3', '--num_iters', '300', ],
            ['--num_rows', '20', '--num_cols', '2', '--num_iters', '300', ],
            ['--num_rows', '20', '--num_cols', '3', '--num_iters', '300', ],
            ]
    configs_list = map(args_to_config, args_list)
    do_experiments(runner, writer, configs_list)

    config_list = read_all_configs()
    results = read_results(config_list[:-1])
{'fraction_non_alphanumeric': 0.10800620980261699, 'fraction_numerical': 0.005988023952095809, 'mean_word_length': 4.195852534562212, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5917369', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1446, 'n_words': 294}