<jupyter_start><jupyter_text>Evolution of CRO disclosure over time<jupyter_code>import sys
import math
from datetime import date
from dateutil.relativedelta import relativedelta

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import matplotlib.dates
from matplotlib.ticker import MaxNLocator
import seaborn as sns

sys.path.append('../..')
from data import constants

# Setup seaborn
sns.set_theme(style="ticks", rc={'text.usetex' : True})
sns.set_context("paper")

# Read main file
df = pd.read_pickle("/Users/david/Nextcloud/Dokumente/Education/Uni Bern/Master Thesis/Analyzing Financial Climate Disclosures with NLP/Data/stoxx_inference/Firm_AnnualReport_Paragraphs_with_actual_back.pkl")
df = df.set_index(["id"])
assert df.index.is_unique, "Index is not unique. Check the data!"

# Read master for scaling
df_master = pd.read_csv("/Users/david/Nextcloud/Dokumente/Education/Uni Bern/Master Thesis/Analyzing Financial Climate Disclosures with NLP/Data/stoxx_inference/Firm_AnnualReport.csv")
df_reports_count = df_master.groupby('year')['is_inferred'].sum()<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>category_labels = constants.cro_category_labels
category_codes = constants.cro_category_codes
colors = sns.color_palette("GnBu", len(category_codes))
levelize_year = 2015<jupyter_output><empty_output><jupyter_text>Evolution over the yearsShows the level of *average number of predicted CRO's per report* (ACROR) over time, in 2015 *levels* (i.e. 2015 scaled to 1).    1. divide by amount of reports in each year    2. then report the levels by dividing by 2015 values    Why 2015? Paris and simply because it of missing values otherwise...<jupyter_code># Create yearly bins for each category
df_years = df.groupby('year')[[f"{c}_predicted" for c in category_codes]].sum().T

# 1. Divide by number of reports in each year
df_years = df_years / df_reports_count

# 2. Divide by the first column to get levels
level_column = df_years[levelize_year]
df_years = df_years.T / level_column
df_years = df_years.T

# Plot
fig, ax = plt.subplots()
plt.stackplot(df_years.columns,df_years, colors=colors, labels=category_labels)
plt.legend(loc='upper left')
plt.xlim(min(df_years.columns), max(df_years.columns))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
plt.show()
fig.savefig('/Users/david/Nextcloud/Dokumente/Education/Uni Bern/Master Thesis/Analyzing Financial Climate Disclosures with NLP/Thesis/figures/levels_acror_years.pdf', format='pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Evolution by country<jupyter_code>index_id = 'country'
results = {}
pd.pivot_table(df_master, values="is_inferred", index=[index_id], columns=['year'], aggfunc=np.sum, fill_value = 0)
def agg_reports(rows, idx_val):
    splits = rows.index[0].split("-")
    year = int(splits[1].split("_")[1])
    if index_id == 'country':
        idx = splits[0].split("_")[0]
        
    else:
        # TODO: Lookup idx from master file
        #idx = splits[1].split("_")[1]
        pass
    
    normalizer = reports_count.query(f"{idx_val} == '{idx}'")[year]
    return np.sum(rows) / int(normalizer)
reports_count = pd.pivot_table(df_master, values="is_inferred", index=[index_id], columns=['year'], aggfunc=np.sum, fill_value = 0)
for c in constants.cro_categories:
    results[c["code"]] = pd.pivot_table(df, values=f"{c['code']}_predicted", index=[index_id], columns=['year'], aggfunc=lambda row: agg_reports(row, index_id), fill_value = 0, dropna=False)

colors = sns.color_palette("GnBu", len(results))

# Plot
fig, ax = plt.subplots()
plt.stackplot(pr.columns, [i.sum() for i in results.values()], colors=colors, labels=constants.cro_category_labels)
plt.legend(title="Countries", loc='upper left')
# plt.xlim(start_date, end_date)
plt.show()
reports_count = pd.pivot_table(df_master, values="is_inferred", index=['country'], columns=['year'], aggfunc=np.sum, fill_value = 0)
reports_count.loc['ch']
def plot_grid_by_group(groups, group_column):
    reports_count = pd.pivot_table(df_master, values="is_inferred", index=[group_column], columns=['year'], aggfunc=np.sum, fill_value = 0)
    
    rows = math.ceil(len(groups) / 4)
    fig, axs = plt.subplots(rows, 4, 
                            figsize=(12, 10 if rows > 1 else 5), 
                            sharex=False, 
                            sharey=True, 
                            # constrained_layout=True
                           )

    axs = axs.ravel()
    for idx, c in enumerate(groups):
        ax = axs[idx]
        
        df_group = df.query(f"{group_column} == @c")
        
        # Create yearly bins for each category
        df_years = df_group.groupby('year')[[f"{c}_predicted" for c in category_codes]].sum().T
        
        # 1. Divide by number of reports in each year
        df_years = df_years / reports_count.loc[c]

        # 2. Divide by the first column to get levels
        level_column = df_years[levelize_year]
        df_years = df_years.T / level_column
        df_years = df_years.T
        
        ax.stackplot(df_years.columns, df_years, labels=category_labels, colors=colors)
        ax.set_xlim(min(df_group.year), max(df_group.year))
        ax.title.set_text(c)

        # for label in ax.xaxis.get_ticklabels()[::2]:
        #    label.set_visible(False)

        # if idx == len(countries) - 1:
        #    handles, labels = ax.get_legend_handles_labels()
        #    fig.legend(handles, labels, title="Governing resource", loc='lower center', ncol=len(gov_resources))

    fig.subplots_adjust(bottom=0.05 if rows > 2 else 0.25)
    return fig

all_countries = sorted(df_master.country.unique())
all_countries_fig = plot_grid_by_group(all_countries, 'country')
# all_countries_fig.savefig('/Users/david/Nextcloud/Dokumente/Education/Uni Bern/Master Thesis/Analyzing Financial Climate Disclosures with NLP/Thesis/figures/levels_acror_countries.pdf', format='pdf', bbox_inches='tight')
selected_countries_fig = plot_grid_by_group(["de", "ch", "fr", "gb"], 'country')<jupyter_output><empty_output><jupyter_text>Industry<jupyter_code>all_industries = sorted(df_master.icb_industry.unique())
all_inudustries_fig = plot_grid_by_group(all_industries, 'icb_industry')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12480051069262688, 'fraction_numerical': 0.00893712097031599, 'mean_word_length': 4.402586206896552, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '425755', 'license': 'MIT', 'n_tokens_mistral': 2256, 'n_tokens_neox': 2125, 'n_words': 445, 'path': 'notebooks/charts/archive/CROEvolutionStacked.ipynb', 'repo_name': 'dafrie/fin-disclosures-nlp', 'size': '6266'}