<commit_before>import numpy as np


def set_first_intercept(w, X1):
    ctr_counts = reduce(
        lambda a, b: (a[0] + b[0], a[1] + b[1]),
        map(
            lambda x: (x[0], 1),
            X1))
    ctr_ratio = float(ctr_counts[0]) / float(ctr_counts[1]) + 10**-6
    new_intercept = -np.log(ctr_ratio**-1 - 1)
    w[0, 0] = new_intercept
    return w


def set_first_intercept_spark(w, X1):
    ctr_counts = X1.map(
        lambda x: (x[0], 1),
        preservesPartitioning=True
    ).fold(
        (0, 0),
        lambda a, b: (a[0] + b[0], a[1] + b[1])
    )
    ctr_ratio = float(ctr_counts[0]) / float(ctr_counts[1]) + 10**-6
    new_intercept = -np.log(ctr_ratio**-1 - 1)
    w[0, 0] = new_intercept
    return w<commit_msg>refactor set_first_intercept for spark for arbitrary sized tensors<commit_after>import numpy as np


def set_first_intercept(w, X1):
    ctr_counts = reduce(
        lambda a, b: (a[0] + b[0], a[1] + b[1]),
        map(
            lambda x: (x[0], 1),
            X1))
    ctr_ratio = float(ctr_counts[0]) / float(ctr_counts[1]) + 10**-6
    new_intercept = -np.log(ctr_ratio**-1 - 1)
    w[0, 0] = new_intercept
    return w


def set_first_intercept_spark(w, X1):
    ctr_counts = X1.map(
        lambda x: (x[0], 1),
        preservesPartitioning=True
    ).fold(
        (0, 0),
        lambda a, b: (a[0] + b[0], a[1] + b[1])
    )
    numer = ctr_counts[0]
    denom = ctr_counts[1]
    if denom == 0:
        denom = 1
    ctr_ratio = numer / denom
    # new_intercept = -ln(ctr_ratio**-1 - 1)
    ctr_ratio.data = -np.log([
        ((_ + 10**-6) ** -1) - 1.
        for _ in ctr_ratio.data])
    new_intercept = ctr_ratio
    w[0] = new_intercept.T
    return w
{'fraction_non_alphanumeric': 0.16072472238457042, 'fraction_numerical': 0.043834015195791935, 'mean_word_length': 2.224105461393597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4858879', 'n_tokens_mistral': 740, 'n_tokens_neox': 721, 'n_words': 155}