<commit_before>"""
Data processing facilities
"""

import collections

import numpy as np
import matplotlib.pylab as plt


def network_density():
    """ Plot network edge density vs correlation quotient
    """
    data = np.load('results/data_cache.npy')

    points = collections.defaultdict(list)
    for syst, mat, sol in data:
        max_edge_num = syst.jacobian.shape[0] * (syst.jacobian.shape[0]-1)
        dens = np.count_nonzero(syst.jacobian) / max_edge_num
        quot = mat[0, 2] / mat[1, 2]
        points[dens].append(quot)

    densities = []
    quotients = []
    errbars = []
    for dens, quots in points.items():
        densities.append(dens)
        quotients.append(np.mean(quots))
        errbars.append(np.std(quots))

    plt.errorbar(
        densities, quotients, yerr=errbars,
        fmt='o', clip_on=False)

    plt.xlabel('motif edge density')
    plt.ylabel('correlation quotient')
    #plt.xlim((0.5, 1.01))

    plt.tight_layout()<commit_msg>Small bugfix<commit_after>"""
Data processing facilities
"""

import collections

import numpy as np
import matplotlib.pylab as plt


def network_density():
    """ Plot network edge density vs correlation quotient
    """
    data = np.load('results/data_cache.npy')

    points = collections.defaultdict(list)
    for syst, mat, sol in data:
        max_edge_num = syst.jacobian.shape[0] * (syst.jacobian.shape[0]-1)
        dens = np.count_nonzero(syst.jacobian) / max_edge_num
        quot = mat[0, 2] / mat[1, 2] if mat[1, 2] != 0 else 0
        points[dens].append(quot)

    densities = []
    quotients = []
    errbars = []
    for dens, quots in points.items():
        densities.append(dens)
        quotients.append(np.mean(quots))
        errbars.append(np.std(quots))

    plt.errorbar(
        densities, quotients, yerr=errbars,
        fmt='o', clip_on=False)

    plt.xlabel('motif edge density')
    plt.ylabel('correlation quotient')
    #plt.xlim((0.5, 1.01))

    plt.tight_layout()
{'fraction_non_alphanumeric': 0.13256048387096775, 'fraction_numerical': 0.014112903225806451, 'mean_word_length': 2.88454011741683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3163744', 'n_tokens_mistral': 707, 'n_tokens_neox': 665, 'n_words': 146}