<commit_before>import itertools
import os
import pickle
import inspect
import numpy as np
from chainconsumer import ChainConsumer

from dessn.models.d_simple_stan.run import get_truths_labels_significance


def get_chain(filename, name_map, replace=True):
    print("Loading chain from %s" % filename)
    try:
        with open(filename, 'rb') as output:
            chain = pickle.load(output)
            if replace:
                del chain["intrinsic_correlation"]
                keys = list(chain.keys())
                for key in keys:
                    if key in name_map:
                        label = name_map[key]
                        if isinstance(label, list):
                            for i, l in enumerate(label):
                                chain[l] = chain[key][:, i]
                        else:
                            chain[label] = chain[key]
                        del chain[key]
                    else:
                        new_key = key.replace("_", r"\_")
                        if new_key != key:
                            chain[new_key] = chain[key]
                            del chain[key]
    except EOFError:
        chain = None
    return chain


def load_stan_from_folder(folder, replace=True, merge=True, cut=False, num=None, max_deviation=3, trim=False, trim_v=-8):
    vals = get_truths_labels_significance()
    full_params = [[k[2]] if not isinstance(k[2], list) else k[2] for k in vals if k[2] is not None]
    params = [[k[2]] if not isinstance(k[2], list) else k[2] for k in vals if
              k[3] and k[2] is not None]
    full_params = list(itertools.chain.from_iterable(full_params))
    full_params.remove("$\\rho$")
    params = list(itertools.chain.from_iterable(params))
    name_map = {k[0]: k[2] for k in vals}
    truths = {k[2]: k[1] for k in vals if not isinstance(k[2], list)}
    is_array = [k[0] for k in vals if not isinstance(k[1], float) and not isinstance(k[1], int)]
    cs = {}
    fs = sorted([f for f in os.listdir(folder) if f.startswith("stan") and f.endswith(".pkl")])
    if num is not None:
        filter = "_%d_" % num
        fs = [f for f in fs if filter in f]

    for f in fs:
        splits = f.split("_")
        c = splits[1]
        t = os.path.abspath(folder + os.sep + f)
        if cs.get(c) is None:
            cs[c] = []
        file_chain = get_chain(t, name_map, replace=replace)
        if file_chain is not None:<commit_msg>4 standard deviations now, to better see differences between approx and full corrections<commit_after>import itertools
import os
import pickle
import inspect
import numpy as np
from chainconsumer import ChainConsumer

from dessn.models.d_simple_stan.run import get_truths_labels_significance


def get_chain(filename, name_map, replace=True):
    print("Loading chain from %s" % filename)
    try:
        with open(filename, 'rb') as output:
            chain = pickle.load(output)
            if replace:
                del chain["intrinsic_correlation"]
                keys = list(chain.keys())
                for key in keys:
                    if key in name_map:
                        label = name_map[key]
                        if isinstance(label, list):
                            for i, l in enumerate(label):
                                chain[l] = chain[key][:, i]
                        else:
                            chain[label] = chain[key]
                        del chain[key]
                    else:
                        new_key = key.replace("_", r"\_")
                        if new_key != key:
                            chain[new_key] = chain[key]
                            del chain[key]
    except EOFError:
        chain = None
    return chain


def load_stan_from_folder(folder, replace=True, merge=True, cut=False, num=None, max_deviation=4, trim=False, trim_v=-9):
    vals = get_truths_labels_significance()
    full_params = [[k[2]] if not isinstance(k[2], list) else k[2] for k in vals if k[2] is not None]
    params = [[k[2]] if not isinstance(k[2], list) else k[2] for k in vals if
              k[3] and k[2] is not None]
    full_params = list(itertools.chain.from_iterable(full_params))
    full_params.remove("$\\rho$")
    params = list(itertools.chain.from_iterable(params))
    name_map = {k[0]: k[2] for k in vals}
    truths = {k[2]: k[1] for k in vals if not isinstance(k[2], list)}
    is_array = [k[0] for k in vals if not isinstance(k[1], float) and not isinstance(k[1], int)]
    cs = {}
    fs = sorted([f for f in os.listdir(folder) if f.startswith("stan") and f.endswith(".pkl")])
    if num is not None:
        filter = "_%d_" % num
        fs = [f for f in fs if filter in f]

    for f in fs:
        splits = f.split("_")
        c = splits[1]
        t = os.path.abspath(folder + os.sep + f)
        if cs.get(c) is None:
            cs[c] = []
        file_chain = get_chain(t, name_map, replace=replace)
        if file_chain is not None:
{'fraction_non_alphanumeric': 0.10954773869346733, 'fraction_numerical': 0.008241206030150754, 'mean_word_length': 1.8713214079630698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5880496', 'n_tokens_mistral': 1564, 'n_tokens_neox': 1516, 'n_words': 461}