<jupyter_start><jupyter_text>This notebook allows you to monitor stats while [train_dupnet.ipynb](train_dupnet.ipynb) is running in a different notebook.<jupyter_code>import os
import ast
from collections import Counter

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sdcdup.utils import get_project_root
from sdcdup.visualization import reversed_recombined_holt_winters
from sdcdup.features import SDCImageContainer

%load_ext dotenv
%dotenv
%matplotlib inline
%reload_ext autoreload
%autoreload 2

SMALL_SIZE = 10
MEDIUM_SIZE = 12
BIGGER_SIZE = 16
BIGGEST_SIZE = 20
plt.rc('font', size=BIGGEST_SIZE)         # controls default text sizes
plt.rc('axes', titlesize=BIGGEST_SIZE)    # fontsize of the axes title
plt.rc('axes', labelsize=BIGGEST_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=BIGGER_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=BIGGER_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGEST_SIZE)  # fontsize of the figure title

project_root = get_project_root()
models_dir = os.path.join(project_root, 'models')
train_image_dir = os.path.join(project_root, os.getenv('RAW_DATA_DIR'), 'train_768')
dataset_filename = 'full_SDC_dataset_from_tiles.csv'
full_dataset_filename = os.path.join(project_root, os.getenv('PROCESSED_DATA_DIR'), dataset_filename)
df = pd.read_csv(full_dataset_filename)
full_dataset = list(zip(*[df[c].values.tolist() for c in df]))
print(len(full_dataset))
nrows = 1
ncols = 2
fname = 'dup_model.2019_0727_2246.metrics.csv'
fname = 'dup_model.2019_0730_1923.metrics.csv'
fname = 'dup_model.2019_0802_2209.metrics.csv'
# fname = 'dup_model.2019_1126_0941.metrics.csv'
df_stats = pd.read_csv(os.path.join(models_dir, fname))

fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(14, 6))
# span = 111//30 = 3
# span = epochs // 100
span = 2
field_ylim = {
    'loss': (0, 0.005), 
    'brier_loss': (0, None),
    'focal_loss': (0, None), 
    'focal_loss_1': (0, 5), 
    'focal_loss_2': (0, 5), 
    'dice_coef_loss': (0, None), 
    'soft_dice_loss': (0, None),
    'binary_crossentropy': (0, None),
    'acc': (None, 1), 
    'iou': (None, 1), 
    'mean_iou': (None, 1), 
    'f2': (None, 1), 
    'fbeta': (None, 1), 
    'soft_dice_coef': (None, 1),
    'hard_dice_coef': (None, 1), 
    'hard_dice_coef2': (None, 1), 
    'hard_dice_coef_ch1': (None, 1),
} 
stats_fields = ('loss', 'acc')
filename = []
for j in range(ncols):
    field = stats_fields[j]
#     filename.append(field)

    stats_dict = {
        'train_'+field: df_stats['train_'+field],
        'val_'+field: df_stats['val_'+field],
        'train_'+field+' (hw)': reversed_recombined_holt_winters(np.array(df_stats['train_'+field]), span=span),
        'val_'+field+' (hw)': reversed_recombined_holt_winters(np.array(df_stats['val_'+field]), span=span)
    }
    alphas = {
        'train_'+field: 0.3,
        'val_'+field: 0.3,
        'train_'+field+' (hw)': 1,
        'val_'+field+' (hw)': 1
    }
    ax = axes[j]
    legend_labels = []
    for key, value in stats_dict.items():
        ax.plot(value, alpha=alphas[key])
        legend_labels.append(key)
    # Special case for f2 callback.
    if field == 'fbeta' and 'val_f2' in df_stats:
        ax.plot(df_stats['val_f2'])
        legend_labels.append('val_f2')
    ax.set_title(field)
    ax.set_xlabel(r'epoch')
    ax.set_xlim((0, len(df_stats['epoch'])))
    ax.set_ylim(field_ylim[field])
    ax.grid(True)
    ax.legend(legend_labels)

plt.tight_layout()
# filename = '-'.join(filename)
# plt.savefig(out_dir + filename + '.png')
plt.show()

nrows = 1
ncols = 2

fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(14, 6))

ignored_runs = [
    "dup_model.2019_0709_0149.metrics.csv",
    "dup_model.2019_0714_0200.metrics.csv",
    "dup_model.2019_0720_0155.metrics.csv",
    "dup_model.2019_0724_0938.metrics.csv",
    "dup_model.2019_0727_2246.metrics.csv",
#     "dup_model.2019_0730_1923.metrics.csv",
    "dup_model.2019_0731_2310.metrics.csv",
    "dup_model.2019_0801_0954.metrics.csv",
    "dup_model.2019_0801_2010.metrics.csv",
    "dup_model.2019_0802_0708.metrics.csv",
#     "dup_model.2019_0802_2209.metrics.csv",
    "dup_model.2019_0805_0015.metrics.csv",
    "dup_model.2019_0805_1708.metrics.csv",
#     "dup_model.2019_1126_0941.metrics.csv",
]
csv_files = []
for fname in os.listdir(models_dir):
    if fname.endswith('metrics.csv') and fname not in ignored_runs:
        csv_files.append(fname)

legend_labels = []
dataframes = []
for fname in sorted(csv_files):
    dataframes.append(pd.read_csv(os.path.join(models_dir, fname)))
    legend_labels.append(fname.split('.')[-3])

xlabels = ('epoch', 'time')
xmax = (100, 100000)
for j in range(ncols):
#     xmax = 0
    xlabel = xlabels[j]
    ax = axes[j]
    for df in dataframes:
#         value = df.train_loss
#         ax.plot(df[xlabel], value)
#         value = reversed_recombined_holt_winters(np.array(df.val_loss), span=2)
        value = df.val_loss
        ax.plot(df[xlabel], value)
#         xmax = max(xmax, max(df[xlabel]))

    ax.set_title('val loss')
    ax.set_xlabel(xlabel)
    ax.set_xlim((0, xmax[j]))
    ax.set_ylim((0, 0.005))
#     ax.grid(True)
    ax.legend(legend_labels)

plt.tight_layout()
plt.show()

fname = "dup_model.2019_0802_2209.121.0.000022.avl.csv"
df_avl = pd.read_csv(os.path.join(models_dir, fname))
df_avl.describe()
df_avl['ages'].hist(bins=df_avl['ages'].max()//100);
df_avl['visits'].hist(bins=df_avl['visits'].max());
run_name = "2019_0802_2209"
csv_files = []
for fname in os.listdir(models_dir):
    if fname.endswith('avl.csv') and run_name in fname:
        csv_files.append(fname)

bad_lines = Counter()
for fname in sorted(csv_files):
    model_name, date_time, epoch_str, val_loss0, val_loss1, ds_type, file_type = fname.split('.')
    epoch = int(epoch_str)
    val_loss = '.'.join([val_loss0, val_loss1])
    df = pd.read_csv(os.path.join(models_dir, fname))
    for bad_line in df.index[df['losses'] > 5].tolist():
        bad_lines[bad_line] += 1
    print(f"{epoch:>3} | {df.ages.max():>5}, ({df.visits.min()}, {df.visits.max():>2}), {df.losses.max():>7.4f}, {df.losses.sum():>10.2f}, {float(val_loss):<8.6f}")
df_ids = pd.read_csv(os.path.join(models_dir, f"dup_model2.{run_name}.ids.csv"))

records = []
for bad_line, ct in sorted(bad_lines.items(), key=lambda x: x[1], reverse=True):
    rec = ast.literal_eval(df_ids.loc[bad_line].img_id)
    if ct > 1:
        records.append(rec)
    if ct > 1:
        print(f"{bad_line:>7} {ct:>2}", rec)
for img1_id, img2_id, idx1, idx2, is_dup in sorted(records):
    print(img1_id, img2_id, idx1, idx2, is_dup)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15796479810678893, 'fraction_numerical': 0.05502144653157817, 'mean_word_length': 4.185582822085889, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '409640', 'license': 'MIT', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2708, 'n_words': 477, 'path': 'notebooks/train_dupnet_sidecar.ipynb', 'repo_name': 'WillieMaddox/Airbus_SDC_dup', 'size': '6761'}