<commit_before>import math
import numpy as np
from functools import reduce

EPS = 1e-4
ZERO = 1e-12
LOG_ZERO = math.log(1e-12)

def normalize2d(vec):
    return (vec.transpose() / vec.sum(axis=1)).transpose()

def normalize1d(vec):
    return vec / vec.sum()

def log_mul(*arrays):
    return np.array(arrays).sum(axis=0)

def log_vec_mul(*vec_arrays):
    return np.sum(vec_arrays, axis=0)

def log_div(*arrays):
    return arrays[0] - log_mul(*arrays[1:])

def log_vec_div(*vec_arrays):
    return vec_arrays[0] - log_vec_mul(*vec_arrays[1:])

def log_add(*arrays):
    a = max(arrays[0], arrays[1])
    b = min(arrays[0], arrays[1])
    return a + math.log(1.0 + math.exp(b-a))

def log_sum(*arrays):
    return reduce(lambda x,y: log_add(x, y), arrays)<commit_msg>[util] log vec mul constant<commit_after>import math
import numpy as np
from functools import reduce

EPS = 1e-4
ZERO = 1e-12
LOG_ZERO = math.log(1e-12)

def normalize2d(vec):
    return (vec.transpose() / vec.sum(axis=1)).transpose()

def normalize1d(vec):
    return vec / vec.sum()

def log_mul(*arrays):
    return np.array(arrays).sum(axis=0)

def log_vec_mul(*vec_arrays):
    return np.sum(vec_arrays, axis=0)

def log_vec_mul_c(array, c):
    return arrays + c

def log_div(*arrays):
    return arrays[0] - log_mul(*arrays[1:])

def log_vec_div(*vec_arrays):
    return vec_arrays[0] - log_vec_mul(*vec_arrays[1:])

def log_vec_div_c(array, c):
    return array - c

def log_add(*arrays):
    a = max(arrays[0], arrays[1])
    b = min(arrays[0], arrays[1])
    return a + math.log(1.0 + math.exp(b-a))

def log_sum(*arrays):
    return reduce(lambda x,y: log_add(x, y), arrays)
{'fraction_non_alphanumeric': 0.1560798548094374, 'fraction_numerical': 0.027828191167574106, 'mean_word_length': 4.234177215189874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1587097', 'n_tokens_mistral': 689, 'n_tokens_neox': 652, 'n_words': 145}