<commit_before>import pylab
import matplotlib
import os
import data
from helper import ensure_dir, get_markers, get_upb_colors, get_preset_colors, label_rename
# ensure correct fonts for ACM/IEEE
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
# style template ('bmh', 'ggplot', 'fivethirtyeight')
#matplotlib.pyplot.style.use("ggplot")
matplotlib.pyplot.style.use("grayscale")


def multi_scenario_plot(
        output,
        ed,
        xfield="numbermb",
        yfield=["pps_global", "pps_local"],
        destinction_field="backend",
        rowfilter={"controldelay": 0},
        xname=None,
        yname="pps",
        name_pre="",
        name_post=""
        ):
    # get dataframe
    df = ed.get_combined_df()
    # input processing
    if isinstance(yfield, basestring) or isinstance(yfield, str):
        yfield = [yfield]
    xname = xfield if xname is None else xname
    yname = yfield if yname is None else yname
    # setup
    pname = "plt_multi_%s%s_%s%s" % (name_pre, xfield, "_".join(yfield), name_post)
    print "Plotting: %s" % pname
    # figure
    fig = pylab.figure()
    g1 = fig.add_subplot(111)
    # iterators
    markers = get_markers()
    colors = get_preset_colors()

    # get different values for lines to be plotted
    destinction_values = [""]
    if destinction_field is not None:
        destinction_values = list(set(df[destinction_field].tolist()))

    print df
    print destinction_values

    # filter data for single plot
    for yf in yfield:
        for dv in destinction_values:
            dfiltered = df
            if destinction_field is not None:
                dfiltered = dfiltered[dfiltered[destinction_field] == dv]
            if len(rowfilter) > 0:
                for k, v in rowfilter.items():
                    dfiltered = dfiltered[dfiltered[k] == v]

            # reduce data to have xfield -> yflied 1:1 relationship
            dfiltered = dfiltered.groupby(xfield).mean()
            print dfiltered
            # do plots
            g1.plot(
                dfiltered.index.values,
                dfiltered[yf].tolist(),
                linewidth=1.5,
                alpha=1.0,
                marker=markers.next(),
                color=colors.next(),
                label=label_rename("%s %s" % (dv, yf)))

    # label etc.
    g1.legend(
        bbox_to_anchor=(1.02, 1),
        loc=2,
        borderaxespad=0.,
        fancybox=False,
        shadow=False,
        ncol=1,
        prop={'size': 10})
    g1.set_xlabel(xname)
    g1.set_ylabel(yname)
    # fig.suptitle(sc.name)
    # store to disc
    fig.set_size_inches(7, 4)<commit_msg>added confidence intervalls to combined plots<commit_after>import pylab
import matplotlib
from scipy import stats as st
import scipy
import scikits.bootstrap as bootstrap
import numpy as np
import os
import data
from helper import ensure_dir, get_markers, get_upb_colors, get_preset_colors, label_rename
# ensure correct fonts for ACM/IEEE
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
# style template ('bmh', 'ggplot', 'fivethirtyeight')
#matplotlib.pyplot.style.use("ggplot")
matplotlib.pyplot.style.use("grayscale")

ENABLE_CONFIDENCE_INTERVAL = True

def multi_scenario_plot(
        output,
        ed,
        xfield="numbermb",
        yfield=["pps_global", "pps_local"],
        destinction_field="backend",
        rowfilter={"controldelay": 0},
        xname=None,
        yname="pps",
        name_pre="",
        name_post=""
        ):
    # get dataframe
    df = ed.get_combined_df()
    # input processing
    if isinstance(yfield, basestring) or isinstance(yfield, str):
        yfield = [yfield]
    xname = xfield if xname is None else xname
    yname = yfield if yname is None else yname
    # setup
    pname = "plt_multi_%s%s_%s%s" % (name_pre, xfield, "_".join(yfield), name_post)
    print "Plotting: %s" % pname
    # figure
    fig = pylab.figure()
    g1 = fig.add_subplot(111)
    # iterators
    markers = get_markers()
    colors = get_preset_colors()

    # get different values for lines to be plotted
    destinction_values = [""]
    if destinction_field is not None:
        destinction_values = list(set(df[destinction_field].tolist()))

    print df
    print destinction_values

    # filter data for single plot
    for yf in yfield:
        for dv in destinction_values:
            dfiltered = df
            if destinction_field is not None:
                dfiltered = dfiltered[dfiltered[destinction_field] == dv]
            if len(rowfilter) > 0:
                for k, v in rowfilter.items():
                    dfiltered = dfiltered[dfiltered[k] == v]

            # reduce data to have xfield -> yflied 1:1 relationship
            dfmean = dfiltered.groupby(xfield).mean()
            print dfmean["pps_global"]
            # calc confidence intervals:
            # http://stackoverflow.com/questions/18039923/standard-error-ignoring-nan-in-pandas-groupby-groups
            # http://www.randalolson.com/2012/08/06/statistical-analysis-made-easy-in-python/
            # http://stackoverflow.com/questions/15033511/compute-a-confidence-interval-from-sample-data
            dfci = dfiltered.groupby(xfield).aggregate(lambda x: scipy.stats.sem(x) * scipy.stats.t.ppf((1+0.95)/2., len(x)-1))
            print dfci["pps_global"]

            # do plots
            if ENABLE_CONFIDENCE_INTERVAL:
                g1.errorbar(
                    dfmean.index.values,
                    dfmean[yf].tolist(),
                    linewidth=1.5,
                    alpha=1.0,
                    marker=markers.next(),
                    color=colors.next(),
                    label=label_rename("%s %s" % (dv, yf)),
                    yerr=dfci[yf].tolist())
            else:
                g1.plot(
                    dfmean.index.values,
                    dfmean[yf].tolist(),
                    linewidth=1.5,
                    alpha=1.0,
                    marker=markers.next(),
                    color=colors.next(),
                    label=label_rename("%s %s" % (dv, yf)))

    # label etc.
    g1.legend(
        bbox_to_anchor=(1.02, 1),
        loc=2,
        borderaxespad=0.,
        fancybox=False,
        shadow=False,
        ncol=1,
        prop={'size': 10})
    g1.set_xlabel(xname)
    g1.set_ylabel(yname)
    # fig.suptitle(sc.name)
    # store to disc
    fig.set_size_inches(7, 4)
{'fraction_non_alphanumeric': 0.10990879579533158, 'fraction_numerical': 0.014994589581078991, 'mean_word_length': 2.1225868725868726, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1996536', 'n_tokens_mistral': 2176, 'n_tokens_neox': 2047, 'n_words': 443}