<jupyter_start><jupyter_text>Min, Mean, Max calulations for the table===<jupyter_code># First table part: overview
df.min()
df.mean()
df.max()
# Second part: study, abstractive
df.groupby("classtype").describe()["manual_r2"]
# Second part: study, abstractive
df.groupby("classtype").describe()["r2"]
df.loc[:,["r2","classtype"]].groupby("classtype").hist()
plt.show()
l = list(df.columns)
for x in l:
    for y in l:
        if l.index(x) >= l.index(y): 
            continue    
        try:
            df.plot.scatter(x=x, y=y)
            plt.show()
        except:
            plt.close()
            print "error on %s with %s" % (x,y)
import seaborn as sns
%matplotlib inline

sns.set()

# Plot tip as a function of toal bill across days
g = sns.lmplot(x="r2", y="manual_r2", hue = "classtype", truncate=False,  data=df, fit_reg=False)

# Use more informative axis labels than are provided by default
g.set_axis_labels("extractive r2", "abstractive r2")

sns.set()

g = sns.FacetGrid(tips, row="classtype", margin_titles=True)
bins = np.linspace(0, 60, 13)
g.map(plt.hist, "total_bill", color="steelblue", bins=bins, lw=0)

sns.kdeplot(data=df.loc[df.classtype.str.contains("r2)
plt.show()
df.loc[:,["manual_r2", "classtype"]].groupby("classtype").describe()
df.describe()<jupyter_output><empty_output><jupyter_text>Calculating t-test significance==== Extractive summary comparison---<jupyter_code>from scipy.stats import ttest_ind

column = "r2"
ttest_ind(df.loc[df.classtype.str.contains("baseline")].loc[:,column], df.loc[df.classtype.str.contains("wegfg")].loc[:,column])
ttest_ind(df.loc[df.classtype.str.contains("baseline")].loc[:,column], df.loc[df.classtype.str.contains("werwfg")].loc[:,column])
ttest_ind(df.loc[df.classtype.str.contains("wegfg")].loc[:,column], df.loc[df.classtype.str.contains("werwfg")].loc[:,column])
df.groupby("classtype")[column].plot.hist(legend =True, alpha=0.6 )
plt.show()
df.groupby("classtype")[column].plot.kde(legend =True)
plt.show()<jupyter_output><empty_output><jupyter_text>Abstractive summaries---kjhadkjadkjahdkasj d* askdjh kj* v<jupyter_code>column = "r2"
ttest_ind(df.loc[df.classtype.str.contains("baseline")].loc[:,column], df.loc[df.classtype.str.contains("wegfg")].loc[:,column], equal_var=False)
ttest_ind(df.loc[df.classtype.str.contains("baseline")].loc[:,column], df.loc[df.classtype.str.contains("werwfg")].loc[:,column], equal_var=False)
ttest_ind(df.loc[df.classtype.str.contains("wegfg")].loc[:,column], 
          df.loc[df.classtype.str.contains("werwfg")].loc[:,column], equal_var=False)
df.groupby("classtype")[column].plot.hist(legend =True, alpha=0.7 )
plt.show()
df.groupby("classtype")[column].plot.kde(legend =True)
plt.show()
df.corr()<jupyter_output><empty_output><jupyter_text>ANOVA-Tests<jupyter_code>initial_summaries = pd.Series([])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19206460674157302, 'fraction_numerical': 0.007373595505617977, 'mean_word_length': 5.931873479318734, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '160314', 'license': 'Apache-2.0', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1135, 'n_words': 158, 'path': 'scripts/user-study-analysis.ipynb', 'repo_name': 'avineshpvs/vldb2018-sherlock', 'size': '2848'}