<jupyter_start><jupyter_text>Data Analysis<jupyter_code>import pandas as pd
import os
import numpy as np
import seaborn as sns
%matplotlib inline<jupyter_output><empty_output><jupyter_text>1. PreparationsThe following code creates the folder for the output tables and figures. We either read the training file or the complete dataset.<jupyter_code>outdir = "../data_analysis"
if not os.path.exists(outdir):
    os.makedirs(outdir)
#df = pd.read_csv("../data/train.tsv", sep="\t")
#dataset = "train"
df = pd.read_csv("../data/all.tsv", sep="\t")
dataset = "all"
df['class_tertiary'] = df['class_tertiary'].map({0: "FALSE", 1: "UNCERTAIN", 2:"TRUE"})
df['class_binary'] = df['class_binary'].map({0: "FALSE", 1: "TRUE"})<jupyter_output><empty_output><jupyter_text>1. Create tables<jupyter_code># Summary of scores annotated per semantic role label
roles = df.groupby(['role_label'])['label']
summary = roles.describe()
csv_file = os.path.join(outdir, f"{dataset}_statistics_scores.csv")
latex_file = os.path.join(outdir, f"{dataset}_latex_statistics_scores.tex")
summary[["count", "mean", "std", "min", "max"]].round(2).to_latex(latex_file)
summary.to_csv(csv_file)
summary
# Frequency of classes (tertiary) per semantic role
df_count_tertiary = pd.DataFrame()
roles = df.groupby(['role_label', 'class_tertiary']).size().reset_index(name='size')
dict_roles = roles.to_dict("records")
for group in dict_roles:
    df_count_tertiary.set_value(group["role_label"], group['class_tertiary'], group["size"])
df_count_tertiary.fillna("-", inplace=True)
csv_file = os.path.join(outdir, f"{dataset}_statistics_classes_tertiary.csv")
latex_file = os.path.join(outdir, f"{dataset}_latex_statistics_classes_tertiary.tex")
df_count_tertiary.to_csv(csv_file)
df_count_tertiary.to_latex(latex_file)
df_count_tertiary
# Frequency of classes (binary) per semantic role
df_count_binary = pd.DataFrame()
roles = df.groupby(['role_label', 'class_binary']).size().reset_index(name='size')
dict_roles = roles.to_dict("records")
for group in dict_roles:
    df_count_binary.set_value(group["role_label"], group['class_binary'], group["size"])
df_count_binary.fillna("-", inplace=True)
csv_file = os.path.join(outdir, f"{dataset}_statistics_classes_binary.csv")
latex_file = os.path.join(outdir, f"{dataset}_latex_statistics_classes_binary.tex")
df_count_tertiary.to_csv(csv_file)
df_count_tertiary.to_latex(latex_file)
df_count_binary<jupyter_output><empty_output><jupyter_text>2. Create plots<jupyter_code># Settings seaborn
sns.set(color_codes=False)
sns.set_style("whitegrid")
np.random.seed(sum(map(ord, "distributions")))
# Plot distribution of scores
scores = df["label"]
sns_plot = sns.distplot(scores, color="black")
fig = sns_plot.get_figure()
fig_file = os.path.join(outdir, f"{dataset}_plot_distribution_scores.pdf")
fig.savefig(fig_file)
# Plot frequency of tertiary classes
scores = df["class_tertiary"]
sns_plot = sns.countplot(x="class_tertiary", data=df, color="grey", order=["FALSE", "UNCERTAIN", "TRUE"])
fig = sns_plot.get_figure()
fig_file = os.path.join(outdir, f"{dataset}_plot_frequency_tertiary_classes.pdf")
fig.savefig(fig_file)
# Plot frequency of tertiary classes
scores = df["class_binary"]
sns_plot = sns.countplot(x="class_binary", data=df, color="grey", order=["FALSE", "UNCERTAIN", "TRUE"])
fig = sns_plot.get_figure()
fig_file = os.path.join(outdir, f"{dataset}_plot_frequency_binary_classes.pdf")
fig.savefig(fig_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15360138848712757, 'fraction_numerical': 0.0026034133641886026, 'mean_word_length': 8.578947368421053, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '628778', 'license': 'Apache-2.0', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1268, 'n_words': 219, 'path': 'code/2-Data_Analysis.ipynb', 'repo_name': 'cltl/positive-interpretations', 'size': '3457'}