<commit_before>
__all__ = ['calc_nbins', 'histpoints']

import numpy as np
import scipy.stats as stats
import scipy as sp

def calc_nbins(x, maximum=150):
    n =  (max(x) - min(x)) / (2 * len(x)**(-1/3) * (np.percentile(x, 75) - np.percentile(x, 25)))
    return min(n, maximum)

def poisson_limits(N, kind, confidence=0.6827):
    alpha = 1 - confidence
    upper = np.zeros(len(N))
    lower = np.zeros(len(N))
    if kind == 'gamma':
        lower = stats.gamma.ppf(alpha / 2, N)
        upper = stats.gamma.ppf(1 - alpha / 2, N + 1)
    if kind == 'sqrt':
        lower = sqrt(N)
        upper = lower
    else:
        raise ValueError('Unknown distribution: {}'.format(errors))
    # clip lower bars
    lower[N==0] = 0
    return N - lower, upper - N

def histpoints(x, nbins=None, xerr=None, yerr='gamma', *kargs, **kwargs):
    import matplotlib.pyplot as plt

    if not nbins:
        nbins = calc_nbins(x)

    h, bins = np.histogram(x, bins=calc_nbins(x), *kargs, **kwargs)
    width = bins[1] - bins[0]
    center = (bins[:-1] + bins[1:]) / 2
    area = sum(h * width)

    if isintance(yerr, basestring):
        yerr = poisson_limits(h, yerr)

    if xerr == 'binwidth':
        xerr = width / 2

    plt.errorbar(center, h, xerr=xerr, yerr=yerr, fmt='o', color='black', markersize=5)

    return center, h, area

<commit_msg>Fix lots of bugs<commit_after>
from __future__ import division
__all__ = ['calc_nbins', 'histpoints']

import numpy as np
import scipy.stats as stats
import scipy as sp
import logging

def calc_nbins(x, maximum=150):
    n =  (max(x) - min(x)) / (2 * len(x)**(-1/3) * (np.percentile(x, 75) - np.percentile(x, 25)))
    return min(n, maximum)

def poisson_limits(N, kind, confidence=0.6827):
    alpha = 1 - confidence
    upper = np.zeros(len(N))
    lower = np.zeros(len(N))
    if kind == 'gamma':
        lower = stats.gamma.ppf(alpha / 2, N)
        upper = stats.gamma.ppf(1 - alpha / 2, N + 1)
    elif kind == 'sqrt':
        lower = sqrt(N)
        upper = lower
    else:
        raise ValueError('Unknown distribution: {}'.format(kind))
    # clip lower bars
    lower[N==0] = 0
    return N - lower, upper - N

def histpoints(x, bins=None, xerr=None, yerr='gamma', normed=False, **kwargs):
    import matplotlib.pyplot as plt

    if bins is None:
        bins = calc_nbins(x)

    h, bins = np.histogram(x, bins=bins)
    width = bins[1] - bins[0]
    center = (bins[:-1] + bins[1:]) / 2
    area = sum(h * width)

    if isinstance(yerr, str):
        yerr = poisson_limits(h, yerr)

    if xerr == 'binwidth':
        xerr = width / 2

    if normed:
        h = h / area
        yerr = yerr / area

    if not 'color' in kwargs:
        kwargs['color'] = 'black'

    if not 'fmt' in kwargs:
        kwargs['fmt'] = 'o'

    if not 'markersize' in kwargs:
        kwargs['markersize'] = 5

    plt.errorbar(center, h, xerr=xerr, yerr=yerr, **kwargs)

    return center, h, area


{'fraction_non_alphanumeric': 0.15287708546135512, 'fraction_numerical': 0.01974804221995233, 'mean_word_length': 2.5698663426488455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5360145', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1080, 'n_words': 299}