<commit_before>from __future__ import division
from __future__ import absolute_import
from builtins import zip
import numpy as np
from numpy import newaxis as na

from . import stats, general

#########################
#  statistical testing  #
#########################

### graphical

def populations_eq_quantile_plot(pop1, pop2, fig=None, percentilecutoff=5):
    pop1, pop2 = stats.flattendata(pop1), stats.flattendata(pop2)
    assert pop1.ndim == pop2.ndim == 1 or \
            (pop1.ndim == pop2.ndim == 2 and pop1.shape[1] == pop2.shape[1]), \
            'populations must have consistent dimensions'
    D = pop1.shape[1] if pop1.ndim == 2 else 1

    # we want to have the same number of samples
    n1, n2 = pop1.shape[0], pop2.shape[0]
    if n1 != n2:
        # subsample, since interpolation is dangerous
        if n1 < n2:
            pop1, pop2 = pop2, pop1
        np.random.shuffle(pop1)
        pop1 = pop1[:pop2.shape[0]]

    def plot_1d_scaled_quantiles(p1,p2,plot_midline=True):
        import matplotlib.pyplot as plt

        # scaled quantiles so that multiple calls line up
        p1.sort(), p2.sort() # NOTE: destructive! but that's cool
        xmin,xmax = general.scoreatpercentile(p1,percentilecutoff), \
                    general.scoreatpercentile(p1,100-percentilecutoff)
        ymin,ymax = general.scoreatpercentile(p2,percentilecutoff), \
                    general.scoreatpercentile(p2,100-percentilecutoff)
        plt.plot((p1-xmin)/(xmax-xmin),(p2-ymin)/(ymax-ymin))

        if plot_midline:
            plt.plot((0,1),(0,1),'k--')
        plt.axis((0,1,0,1))

    if D == 1:
        if fig is None:<commit_msg>import plt to fix #26<commit_after>from __future__ import division
from __future__ import absolute_import
from builtins import zip
import numpy as np
from numpy import newaxis as na

from . import stats, general

#########################
#  statistical testing  #
#########################

### graphical

def populations_eq_quantile_plot(pop1, pop2, fig=None, percentilecutoff=5):
    import matplotlib.pyplot as plt

    pop1, pop2 = stats.flattendata(pop1), stats.flattendata(pop2)
    assert pop1.ndim == pop2.ndim == 1 or \
            (pop1.ndim == pop2.ndim == 2 and pop1.shape[1] == pop2.shape[1]), \
            'populations must have consistent dimensions'
    D = pop1.shape[1] if pop1.ndim == 2 else 1

    # we want to have the same number of samples
    n1, n2 = pop1.shape[0], pop2.shape[0]
    if n1 != n2:
        # subsample, since interpolation is dangerous
        if n1 < n2:
            pop1, pop2 = pop2, pop1
        np.random.shuffle(pop1)
        pop1 = pop1[:pop2.shape[0]]

    def plot_1d_scaled_quantiles(p1,p2,plot_midline=True):

        # scaled quantiles so that multiple calls line up
        p1.sort(), p2.sort() # NOTE: destructive! but that's cool
        xmin,xmax = general.scoreatpercentile(p1,percentilecutoff), \
                    general.scoreatpercentile(p1,100-percentilecutoff)
        ymin,ymax = general.scoreatpercentile(p2,percentilecutoff), \
                    general.scoreatpercentile(p2,100-percentilecutoff)
        plt.plot((p1-xmin)/(xmax-xmin),(p2-ymin)/(ymax-ymin))

        if plot_midline:
            plt.plot((0,1),(0,1),'k--')
        plt.axis((0,1,0,1))

    if D == 1:
        if fig is None:
{'fraction_non_alphanumeric': 0.14053566054769787, 'fraction_numerical': 0.04092687330725248, 'mean_word_length': 2.811926605504587, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1436234', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1110, 'n_words': 265}