<commit_before>"""Miscellaneous utility functions."""
import itertools
import numpy as np
import pandas as pd
from scipy import stats


def df_to_struct(df):
    """Converts a DataFrame to RPy-compatible structured array."""
    struct_array = df.to_records()
    arr_dtype = struct_array.dtype.descr
    for i, dtype in enumerate(arr_dtype):
        if dtype[1] == np.dtype('object'):
            arr_dtype[i] = (dtype[0], dtype[1].replace("|O", "|S"))

    struct_array = np.asarray([tuple(d) for d in struct_array],
                              dtype=arr_dtype)

    return struct_array


def df_ttest(df, by, key, paired=False, nice=True, **kwargs):
    """Perform a T-test over a DataFrame groupby."""
    test_kind = "rel" if paired else "ind"
    test_func = getattr(stats, "ttest_" + test_kind)
    args = [d[key] for i, d in df.groupby(by)]
    t, p = test_func(*args, **kwargs)
    dof = len(df) - 1 if paired else len(df) - 2
    if nice:
        return "t(%d) = %.3f; p = %.3g%s" % (dof, t, p, sig_stars(p))
    else:
        return pd.Series([t, p], ["t", "p"])


def df_oneway(df, by, key, nice=True, **kwargs):
    """Perform a oneway analysis over variance on a DataFrame groupby."""
    args = [d[key] for i, d in df.groupby(by)]
    f, p = stats.f_oneway(*args, **kwargs)
    dof_b = len(args) - 1
    dof_w = len(df) - dof_b
    if nice:
        return "F(%d, %d) = %.3f; p = %.3g%s" % (dof_b, dof_w, f,
                                                 p, sig_stars(p))<commit_msg>Fix reported ttest dof (not used)<commit_after>"""Miscellaneous utility functions."""
import itertools
import numpy as np
import pandas as pd
from scipy import stats


def df_to_struct(df):
    """Converts a DataFrame to RPy-compatible structured array."""
    struct_array = df.to_records()
    arr_dtype = struct_array.dtype.descr
    for i, dtype in enumerate(arr_dtype):
        if dtype[1] == np.dtype('object'):
            arr_dtype[i] = (dtype[0], dtype[1].replace("|O", "|S"))

    struct_array = np.asarray([tuple(d) for d in struct_array],
                              dtype=arr_dtype)

    return struct_array


def df_ttest(df, by, key, paired=False, nice=True, **kwargs):
    """Perform a T-test over a DataFrame groupby."""
    test_kind = "rel" if paired else "ind"
    test_func = getattr(stats, "ttest_" + test_kind)
    args = [d[key] for i, d in df.groupby(by)]
    t, p = test_func(*args, **kwargs)
    dof = (len(df) / 2) - 1 if paired else len(df) - 2
    if nice:
        return "t(%d) = %.3f; p = %.3g%s" % (dof, t, p, sig_stars(p))
    else:
        return pd.Series([t, p], ["t", "p"])


def df_oneway(df, by, key, nice=True, **kwargs):
    """Perform a oneway analysis over variance on a DataFrame groupby."""
    args = [d[key] for i, d in df.groupby(by)]
    f, p = stats.f_oneway(*args, **kwargs)
    dof_b = len(args) - 1
    dof_w = len(df) - dof_b
    if nice:
        return "F(%d, %d) = %.3f; p = %.3g%s" % (dof_b, dof_w, f,
                                                 p, sig_stars(p))
{'fraction_non_alphanumeric': 0.15554821664464993, 'fraction_numerical': 0.006935270805812417, 'mean_word_length': 2.60166468489893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '595874', 'n_tokens_mistral': 1115, 'n_tokens_neox': 1096, 'n_words': 319}