<jupyter_start><jupyter_text>Config<jupyter_code>category_level = "cro" # ["cro", "cro_sub_type"]
categories = constants.cro_categories if category_level == "cro" else constants.cro_sub_categories
cro_category_labels = [c["label"] for c in categories]

export_dir = os.path.join("/Users/david/Nextcloud/Dokumente/Education/Uni Bern/Master Thesis/Analyzing Financial Climate Disclosures with NLP/ICML2021 Financial Disclosures/figures")

colors = [c["color"] for c in categories]<jupyter_output><empty_output><jupyter_text>Descriptive stats<jupyter_code>df = df.query("country == country")
df_master = df_master.query("country == country")
len(df_master.company.unique())
missing_inferred_reports = set(df_master.index) - set(df.report_id)
print(f"Total number of reports: {len(df_master.index)}, processed reports: {len(df.report_id.unique())}. Missing reports:")
#missing_inferred_reports
print(f'Unique pages: {len(df.groupby(["report_id", "page_no"]).count())}, paragraphs: {len(df)}')
positive_docs = df[df[constants.cro_sub_category_labels].any(axis=1)].copy()
#positive_docs["processed_docs"] = positive_docs.apply(lambda x: dataframe_preparation.spacy_tokenizer(x['text']), axis=1)
print(f"Positive paragraphs: {len(positive_docs)}")

print(f"Total labels : {positive_docs[constants.cro_sub_category_labels].sum().sum()}")
print("Labels by sub category: ")
positive_docs[constants.cro_sub_category_labels].sum()
df_descriptive = pd.DataFrame()
df_tmp = df.groupby("report_id")[constants.cro_sub_category_labels].sum()
df_descriptive["", "Coverage"] = df_tmp.replace(0, np.nan).count()

df_descriptive["Frequency per report", "Mean"] = df_tmp.replace(0, np.nan).mean()
df_descriptive["Frequency per report", "St. Dev."] = df_tmp.replace(0, np.nan).std()
df_descriptive["Frequency per report", "Max"] = df_tmp.max()

"""
df_descriptive["Avg. Lexical diversity per paragraph", "Words"] = df_tmp.apply(lambda x: positive_docs[positive_docs[x.name] > 0].nwords.mean()) 
df_descriptive["Avg. Lexical diversity per paragraph", "Terms"] = df_tmp.apply(lambda x: positive_docs[positive_docs[x.name] > 0].nterms.mean()) 
df_descriptive["Avg. Lexical diversity per paragraph", "TTR"] = df_tmp.apply(lambda x: positive_docs[positive_docs[x.name] > 0].ttr.mean()) 
df_descriptive["Avg. Lexical diversity per paragraph", "MTLD"] = df_tmp.apply(lambda x: positive_docs[positive_docs[x.name] > 0].mtld.mean())
"""

# Add tempory
df_descriptive["Main"] = [constants.map_to_field()[c["parent"]] for c in constants.cro_sub_categories]

df_descriptive.set_index('Main', append=True, inplace=True)
df_descriptive.index.set_names(['second', 'first'], inplace=True)
df_descriptive = df_descriptive.reorder_levels(['first', 'second'])
df_descriptive.index.set_names(['', ''], inplace=True)
df_descriptive = df_descriptive.round(2)
latex_str = tables.export_to_latex(df_descriptive, 
                       filename="pos_paragraphs_descriptive_stats_v2.tex",
                       correct_multicolumn=True,
                       make_bold_row_at=[3],
                       # add_verticalrule_at=[3, 6],
                       index=True, 
                       multirow=False, 
                       multicolumn=False, 
                       bold_rows=True, 
                       multicolumn_format="l", 
                       escape=False,
                       float_format="{:0.2f}".format
                      )
df_descriptive
latex_str<jupyter_output><empty_output><jupyter_text>Evolution over the yearsShows the level of *average number of predicted CR's per report* (ACRR) over time.<jupyter_code>import seaborn as sns

def convert_to_long(df, labels):
    df_long = df.groupby(['Year', 'report_id']).sum()[labels]
    df_long = df_long.reset_index()
    df_long = pd.melt(df_long, id_vars=["Year"], value_vars=labels, var_name='Category', value_name='Average \# mentions (per report)')
    return df_long

def plot_evolution(df, categories, **kwargs):
    colors = [c["color"] for c in categories]
    df = convert_to_long(df, [c["label"] for c in categories])
    ax = sns.lineplot(data=df, x="Year", y="Average \# mentions (per report)", hue="Category", palette=colors, style="Category", **kwargs)
    plt.legend(loc='upper left')
    plt.xlim()
    plt.xlim(min(df.Year),2020)  # max(df.Year))
    ax.set_xlabel("")
    return ax


def plot_evolution_categories(df, categories, **kwargs):
    ax = plot_evolution(df, categories, **kwargs)
    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
    fig = ax.get_figure()
    return fig


def plot_grid(groups, column, categories, exclude_groups=[], ncols=4, **kwargs):
    nrows = math.ceil(len(groups) / ncols)    
    fig = plt.figure(figsize=(12, 15 if nrows > 1 else 5))

    shared_y_ax = None
    shared_x_ax = None
    for idx, c in enumerate(groups):
        if c in exclude_groups:
            is_excluded = True 
            sharey = None
        else:
            is_excluded = False
            sharey = shared_y_ax
        ax = fig.add_subplot(nrows, ncols, idx + 1, sharey=sharey, sharex=shared_x_ax)
        if not shared_y_ax:
            shared_x_ax = ax
        if not is_excluded:
            shared_y_ax = ax
        
        filtered_df = df.query(f"{column} == @c")
        ax2 = plot_evolution(filtered_df, categories, ax=ax, **kwargs)
        ax2.title.set_text(c)
        
        if ax.is_last_row():
            pass
        else:
            plt.setp(ax.get_xticklabels(), visible=False)
            ax.set_xlabel(None)

        if ax.is_first_col():
            pass
        else:
            plt.setp(ax.get_yticklabels(), visible=False)
            ax.set_ylabel(None)
        
        if is_excluded:
            plt.setp(ax.get_yticklabels(), visible=True)
            ax.yaxis.tick_right()
            
    return fig         
tex_fonts = {
    # Use LaTeX to write all text
    "text.usetex": True,
    "font.family": "serif",
    "font.serif": "Times New Roman",
    # Use 10pt font in plots, to match 10pt font in document
    "axes.labelsize": 20,
    "font.size": 20,
    # Make the legend/label fonts a little smaller
    "legend.fontsize": 18,
    "xtick.labelsize": 18,
    "ytick.labelsize": 18,
}

plt.rcParams.update(tex_fonts)

fig1 = plot_evolution_categories(df, constants.cro_categories)
fig1.savefig(os.path.join(export_dir, "levels_acror_cro_years.pdf"), format='pdf', bbox_inches='tight')
fig2 = plot_evolution_categories(df, constants.cro_sub_categories, ci=None)
fig2.axes[0].set_ylabel("")
fig2.savefig(os.path.join(export_dir, f"levels_acror_cro_sub_type_years.pdf"), format='pdf', bbox_inches='tight')
all_countries = sorted(df_master.country.unique())
all_countries_fig = plot_grid(all_countries, 'country', constants.cro_categories, exclude_groups=["Norway", "Spain"], ncols=3, ci=None)
all_countries_fig.savefig(os.path.join(export_dir, f"levels_acror_cro_countries.pdf"), format='pdf', bbox_inches='tight')
tex_fonts = {
    # Use LaTeX to write all text
    "text.usetex": True,
    "font.family": "serif",
    "font.serif": "Times New Roman",
    # Use 10pt font in plots, to match 10pt font in document
    "axes.labelsize": 14,
    "font.size": 14,
    # Make the legend/label fonts a little smaller
    "legend.fontsize": 14,
    "xtick.labelsize": 16,
    "ytick.labelsize": 16,
        "axes.titlesize": 18,
}

plt.rcParams.update(tex_fonts)


selected_countries_fig = plot_grid(["Germany", "Switzerland", "France", "United Kingdom"], 'country', constants.cro_categories, ncols=4, ci=None)
selected_countries_fig.savefig(os.path.join(export_dir, f"levels_acror_cro_selected_countries.pdf"), format='pdf', bbox_inches='tight')
all_industries = sorted(df_master.icb_industry.unique())
all_industries_fig = plot_grid(all_industries, 'icb_industry', constants.cro_categories, exclude_groups=["Energy", "Basic Materials", "Materials"], ncols=4, ci=None)
all_industries_fig.savefig(os.path.join(export_dir, f"levels_acror_cro_industry.pdf"), format='pdf', bbox_inches='tight')
selected_industries_fig = plot_grid(["Consumer Discretionary", "Financials", "Telecommunications", "Energy"], 'icb_industry', constants.cro_categories, exclude_groups=["Energy"], ncols=4, ci=None)
selected_industries_fig.savefig(os.path.join(export_dir, f"levels_acror_cro_selected_industries.pdf"), format='pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Frequency distributions<jupyter_code>from_year = 2015
labels = [c['label'] for c in constants.cro_categories]
colors = [c["color"] for c in constants.cro_categories]

df_long = df.query("Year >= @from_year").groupby(['country', 'report_id']).sum()[labels]
df_long = df_long.reset_index()
print(f"Number of reports (from {from_year}): {len(df_long.report_id.unique())}")
df_long = pd.melt(df_long, id_vars=["country"], value_vars=labels, var_name='Category', value_name='Average \# mentions (per report)')
df_long = df_long.rename(columns={"country": "Country"})

plt.figure(figsize=(9,6))
ax = sns.boxenplot(y="Country", x="Average \# mentions (per report)", hue="Category", data=df_long, palette=colors)
fig = ax.get_figure()
# plt.tight_layout()
fig.subplots_adjust(left=0.2)
fig.savefig(os.path.join(export_dir, f"cro_country_distribution.pdf"), format='pdf')
tex_fonts = {
    # Use LaTeX to write all text
    "text.usetex": True,
    "font.family": "serif",
    "font.serif": "Times New Roman",
    # Use 10pt font in plots, to match 10pt font in document
    "axes.labelsize": 14,
    "font.size": 14,
    # Make the legend/label fonts a little smaller
    "legend.fontsize": 14,
    "xtick.labelsize": 16,
    "ytick.labelsize": 16,
    "axes.titlesize": 18,
}

plt.rcParams.update(tex_fonts)

df_long = df.query("Year >= @from_year").groupby(['icb_industry', 'report_id']).sum()[labels]
df_long = df_long.reset_index()
print(f"Number of reports (from {from_year}): {len(df_long.report_id.unique())}")
df_long = pd.melt(df_long, id_vars=["icb_industry"], value_vars=labels, var_name='Category', value_name='Average \# mentions (per report)')
df_long = df_long.rename(columns={"icb_industry": "Industry"})

plt.figure(figsize=(9,6))
ax = sns.boxenplot(y="Industry", x="Average \# mentions (per report)", hue="Category", data=df_long, palette=colors)
fig = ax.get_figure()
#plt.tight_layout()
fig.subplots_adjust(left=0.2)
fig.savefig(os.path.join(export_dir, f"cro_industry_distribution.pdf"), format='pdf')<jupyter_output>Number of reports (from 2015): 1616<jupyter_text>Preparation<jupyter_code>pr_doc = positive_docs[positive_docs[constants.cro_category_labels[0]]]
tr_doc = positive_docs[positive_docs[constants.cro_category_labels[1]]]
acute_doc = positive_docs[positive_docs[constants.cro_sub_category_labels[0]] > 0]
chron_doc = positive_docs[positive_docs[constants.cro_sub_category_labels[1]] > 0]
policy_doc = positive_docs[positive_docs[constants.cro_sub_category_labels[2]] > 0]
market_doc = positive_docs[positive_docs[constants.cro_sub_category_labels[3]] > 0]
reputation_doc = positive_docs[positive_docs[constants.cro_sub_category_labels[4]] > 0]<jupyter_output><empty_output><jupyter_text>Most frequent ngrams<jupyter_code>from nltk import ngrams, FreqDist

def get_word_counts(docs, most_common=50):
    counts = dict()
    for size in 1, 2:
        counts[size] = FreqDist(ngrams(docs, size))
        
    counts_df = pd.DataFrame(data=counts[1].most_common(most_common), columns=["Unigram", "Frequency"]) # columns=pd.MultiIndex.from_tuples([('Unigram', ''), ('Unigram', 'Frequency')]))
    # counts_df = counts_df.sort_values(by=['Frequency'], ascending=False)
    counts_df = counts_df.join(
        pd.DataFrame(data=counts[2].most_common(most_common), columns=['Bigram', "Frequency2"]) # columns=pd.MultiIndex.from_tuples([('Bigram', ''), ('Bigram', 'Frequency')]))
    )
    counts_df = counts_df.head(most_common)
    
    counts_df['Unigram'] = counts_df['Unigram'].apply(lambda x: " ".join(x))
    counts_df['Bigram'] = counts_df['Bigram'].apply(lambda x: " ".join(x))
    counts_df = counts_df.rename(columns={"Frequency2": "Frequency"})
    return counts_df
    

most_frequent_df = get_word_counts(positive_docs['processed_docs'].explode(), most_common=20)
tables.export_to_latex(most_frequent_df, filename="most_frequent_terms.tex", make_bold_row_at=2, index=False)<jupyter_output><empty_output><jupyter_text>Topic modelling<jupyter_code>import gensim

dictionary = gensim.corpora.Dictionary(positive_docs["processed_docs"])
count = 0
for k, v in dictionary.iteritems():
    print(k, v)
    count += 1
    if count > 10:
        break
dictionary.filter_extremes(no_below=15, no_above=0.9, keep_n= 100000)<jupyter_output><empty_output><jupyter_text>Word Cloud<jupyter_code>from wordcloud import WordCloud
import matplotlib.pyplot as plt

def gen_wordcloud(docs, export_path, **params):
    input_text = " ".join(docs)
    wordcloud = WordCloud(background_color="white", relative_scaling=0.6, **params).generate(input_text)
    plt.figure()
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
    plt.show()
    if export_path:
        wordcloud.to_file(export_path + ".pdf")
        svg = wordcloud.to_svg()
        # with open(export_path + ".svg", "wt") as f:
        #     f.write(svg)

wordclouds_path = os.path.join(export_dir, "wordclouds")
gen_wordcloud(pr_doc["text"], os.path.join(wordclouds_path, "pr"), scale=1, height=400, width=800)
gen_wordcloud(tr_doc["text"], os.path.join(wordclouds_path, "tr"), scale=1, height=400, width=800)

for c in constants.cro_sub_categories:
    docs = positive_docs[positive_docs[c["label"]] > 0]
    path = os.path.join(wordclouds_path, c["code"].lower())
    wordcloud = gen_wordcloud(docs["text"], path, scale=1, height=400, width=600)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14611074728952922, 'fraction_numerical': 0.012442698100851343, 'mean_word_length': 4.8810440735986305, 'pattern_counts': {'":': 29, '<': 32, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '432266', 'license': 'MIT', 'n_tokens_mistral': 5020, 'n_tokens_neox': 4717, 'n_words': 835, 'path': 'notebooks/InferenceResults.ipynb', 'repo_name': 'dafrie/fin-disclosures-nlp', 'size': '13743'}