<commit_before>import numpy

def weighted_mean_and_std(x, w):
    """Return the mean and standard deviation of the data points x, weighted by
    the weights in w (which do not need to sum to 1)."""
    w = numpy.asarray(w, dtype=float)
    x = numpy.asarray(x)
    total_weight = w.sum()
    weighted_mean = (w*x).sum()/total_weight
    squared_diff = (x - weighted_mean)**2
    weighted_var = (w * squared_diff).sum() / total_weight
    return weighted_mean, numpy.sqrt(weighted_var)

def weighted_mean(x, w):
    """Return the mean of the data points x, weighted by
    the weights in w (which do not need to sum to 1)."""
    w = numpy.asarray(w, dtype=float)
    x = numpy.asarray(x)
    return (w*x).sum()/w.sum()
<commit_msg>add gaussian smoothing<commit_after>import numpy

def weighted_mean_and_std(x, w):
    """Return the mean and standard deviation of the data points x, weighted by
    the weights in w (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    weighted_mean = (w*x).sum()
    squared_diff = (x - weighted_mean)**2
    weighted_var = (w * squared_diff).sum()
    return weighted_mean, numpy.sqrt(weighted_var)

def weighted_mean(x, w):
    """Return the mean of the data points x, weighted by the weights in w
    (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    return (w*x).sum()

def _gaussian(x, mu=0, sigma=1):
  return ( (1/numpy.sqrt(2 * numpy.pi * sigma**2) * numpy.exp(-0.5 * ((numpy.asarray(x)-mu)/sigma)**2)) )

def gaussian_mean(x, y, p, std=1):
    """Given a set of positions x where values y were observed, calculate
    the gaussian-weighted mean of those values at a set of new positions p,
    where the gaussian has a specied standard deviation.
    """
  return numpy.array([weighted_mean(y, _gaussian(x, mu=pp, sigma=std)) for pp in p])


{'fraction_non_alphanumeric': 0.1309337506520605, 'fraction_numerical': 0.00782472613458529, 'mean_word_length': 3.3789954337899544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4988716', 'n_tokens_mistral': 688, 'n_tokens_neox': 649, 'n_words': 222}