<commit_before>"""
Usage: `python quantiles.py CONFIG <->BASENAME<:column>...

Supported configuration options:
- column (default: `rebased`)
- yearsets (default: `no`)
- years (default: `null`)
- regions (default: `null`)
- results-root
- output-dir
- do-montecarlo
- only-rcp
- only-models (default: `all`)
- deltamethod (default: `no`) -- otherwise, results root for deltamethod
- file-organize (default: rcp, ssp)
- do-gcmweights (default: true)
- evalqvals (default: ['mean', .17, .5, .83])
- ignore-missing (default: no)
"""

import os, sys, csv, traceback, yaml, copy
import numpy as np

from lib import results, bundles, weights, weights_vcv, configs

config, argv = configs.consume_config()
configs.handle_multiimpact_vcv(config)

do_gcmweights = config.get('do-gcmweights', True)
evalqvals = config.get('evalqvals', ['mean', .17, .5, .83])
output_format = config.get('output-format', 'edfcsv')

columns, basenames, transforms, vectransforms = configs.interpret_filenames(argv, config)

# Collect all available results
data = results.sum_into_data(config['results-root'], basenames, columns, config, transforms, vectransforms)
if configs.is_parallel_deltamethod(config):
    # corresponds to each value in data, if doing parallel deltamethod
    config2 = copy.copy(config)
    config2['deltamethod'] = True
    parallel_deltamethod_data = results.sum_into_data(config['deltamethod'], basenames, columns, config2, transforms, vectransforms)

for filestuff in data:
    print "Creating file: " + str(filestuff)
    with open(configs.csv_makepath(filestuff, config), 'w') as fp:
        writer = csv.writer(fp, quoting=csv.QUOTE_MINIMAL)
        rownames = configs.csv_rownames(config)

        if output_format == 'edfcsv':
            writer.writerow(rownames + map(lambda q: q if isinstance(q, str) else 'q' + str(int(q * 100)), evalqvals))
            encoded_evalqvals = weights.WeightedECDF.encode_evalqvals(evalqvals)
        elif output_format == 'valuescsv':<commit_msg>added in the following logic: if a key (rcp,ssp) or (region, ssp) which exists in the point estimate output does not exist in the delta method output do not output a file for the key and skip to the next iteration of the loop<commit_after>"""
Usage: `python quantiles.py CONFIG <->BASENAME<:column>...

Supported configuration options:
- column (default: `rebased`)
- yearsets (default: `no`)
- years (default: `null`)
- regions (default: `null`)
- results-root
- output-dir
- do-montecarlo
- only-rcp
- only-models (default: `all`)
- deltamethod (default: `no`) -- otherwise, results root for deltamethod
- file-organize (default: rcp, ssp)
- do-gcmweights (default: true)
- evalqvals (default: ['mean', .17, .5, .83])
- ignore-missing (default: no)
"""

import os, sys, csv, traceback, yaml, copy
import numpy as np

from lib import results, bundles, weights, weights_vcv, configs

config, argv = configs.consume_config()
configs.handle_multiimpact_vcv(config)

do_gcmweights = config.get('do-gcmweights', True)
evalqvals = config.get('evalqvals', ['mean', .17, .5, .83])
output_format = config.get('output-format', 'edfcsv')

columns, basenames, transforms, vectransforms = configs.interpret_filenames(argv, config)

# Collect all available results
data = results.sum_into_data(config['results-root'], basenames, columns, config, transforms, vectransforms)
if configs.is_parallel_deltamethod(config):
    # corresponds to each value in data, if doing parallel deltamethod
    config2 = copy.copy(config)
    config2['deltamethod'] = True
    parallel_deltamethod_data = results.sum_into_data(config['deltamethod'], basenames, columns, config2, transforms, vectransforms)

for filestuff in data:
    print "Creating file: " + str(filestuff)
    if configs.is_parallel_deltamethod(config):
        try:
            output_check = parallel_deltamethod_data[filestuff].keys()
        except:
            print str(filestuff) + " is not in delta method output. Skipping model specification..."
            continue
    with open(configs.csv_makepath(filestuff, config), 'w') as fp:
        writer = csv.writer(fp, quoting=csv.QUOTE_MINIMAL)
        rownames = configs.csv_rownames(config)

        if output_format == 'edfcsv':
            writer.writerow(rownames + map(lambda q: q if isinstance(q, str) else 'q' + str(int(q * 100)), evalqvals))
            encoded_evalqvals = weights.WeightedECDF.encode_evalqvals(evalqvals)
        elif output_format == 'valuescsv':
{'fraction_non_alphanumeric': 0.12528064660978896, 'fraction_numerical': 0.007184553210597216, 'mean_word_length': 4.43956043956044, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '628177', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1436, 'n_words': 392}