<jupyter_start><jupyter_text>Train Model<jupyter_code>config = load_config()
dataset = "pascal_5i"
fold = 1
#leo = train_model(config, dataset, fold)<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>mode = "meta_val"
dataloader = PascalDatagenerator(dataset, data_type=mode)
img_transformer = dataloader.transform_image
mask_transformer = dataloader.transform_mask
transformers = (img_transformer, mask_transformer)
val_meta_data = dataloader.get_batch_data()
class_in_metadata = val_meta_data[4]
print(f"classes in mode {mode}: {class_in_metadata}")
def unnormalize_img(img):
    mean = np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1)
    std = np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1)
    img = img * std + mean
    return img
select_class = 1
print("class", class_in_metadata[select_class], "is selected")
batch_data = get_named_dict(val_meta_data, select_class)
num_of_imgs = 5
imgs = list_to_numpy(batch_data.val_imgs[:num_of_imgs], img_transformer)
imgs = unnormalize_img(imgs)
imgs = np.transpose(imgs, (0, 2, 3, 1))
groundtruth = list_to_numpy(batch_data.val_masks[:num_of_imgs], mask_transformer)
prediction, leo, transformers = \
        evaluate_model(dataset, batch_data.tr_imgs, batch_data.tr_masks, batch_data.val_imgs[:num_of_imgs], dataloader)
def plot_batch_mask(imgs, groundtruth, prediction):
    fig = plt.figure(figsize=(20,16))
    imgs = np.expand_dims(imgs, 0) if imgs.ndim == 3 else imgs
    groundtruth = np.expand_dims(groundtruth, 0) if groundtruth.ndim == 2 else groundtruth
    prediction = np.expand_dims(prediction, 0) if prediction.ndim == 2 else prediction
    
    m = len(imgs)
    for i in range(0, m):
        fig.add_subplot(m, 3, i*3+1)
        plt.imshow(imgs[i])
        plt.title("raw images")
        
        fig.add_subplot(m, 3, i*3+2)
        plt.imshow(groundtruth[i], cmap="gray")
        plt.title("groundtruth")
        
        fig.add_subplot(m, 3, i*3+3)
        plt.imshow(prediction[i], cmap="gray")
        plt.title("prediction")
        
    plt.subplots_adjust(hspace=0.5)
    plt.show()
    return
plot_batch_mask(imgs, groundtruth, prediction)
data_path = "./leo_segmentation/data/models/metatrain_results.csv"
df = pd.read_csv(data_path, delimiter=",")
df.head()
plt.plot(df["Fold 0"])
df = df.iloc[:, :-2]
#df = pd.concat([df, ])
#df = df.reset_index().rename(columns={'index':'epochs'})
stats = pd.concat([df.mean(axis=1), df.std(axis=1)], axis=1)
stats = stats.reset_index().rename(columns={'index':'epochs', 0:'Loss', 1:'std_deviation'})
stats
sns.set_style("whitegrid")
font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 50}
matplotlib.rc('font', **font)
means = stats.Loss.values
std = stats.std_deviation.values
lower = means - std
upper = means + std
fig = plt.figure(figsize=(30,12))
plt.fill_between(np.arange(len(means)), lower, upper, color="#0090c1", alpha=0.75)
sns.lineplot(data=stats, x="epochs", y="Loss", color="#051014", label="meta-train loss")
plt.title("Meta-train Loss", fontsize=30)
experiment_number = config.experiment.number
experiment_description = config.experiment.description
experiment_root_path = os.path.join("leo_segmentation", "data", "models", f"experiment_{experiment_number}")
cached_files = os.listdir(experiment_root_path)
cached_files
filename = cached_files[4]
val_ious = pd.read_pickle(os.path.join(experiment_root_path, filename)).T
print(f"filename: {filename}")
filtered_scores = []
for _class, _df in val_ious.iterrows():
    if _class == "episode":
        continue
    temp = [i for i in _df if not np.isnan(i)]
    class_sampling_frequency = len(temp)
    mean_of_all_ious_per_class = np.mean(temp)
    filtered_scores.append((_class, mean_of_all_ious_per_class, class_sampling_frequency))
filtered_val_ious = pd.DataFrame(filtered_scores, columns=["classes", "mean_val_ious", "class_sampling_frequency"])
filtered_val_ious = filtered_val_ious.sort_values(by="mean_val_ious", ascending=False)
filtered_val_ious[filtered_val_ious.mean_val_ious > 0.7].count()
filtered_val_ious
os.listdir("leo_segmentation/data/fss1000/images")[600:800]
filtered_val_ious.class_sampling_frequency.max()
train_ious = pd.read_pickle(os.path.join(experiment_root_path, cached_files[2])).T
filtered_scores = []
for _class, _df in train_ious.iterrows():
    if _class == "episode":
        continue
    temp = [i for i in _df if not np.isnan(i)]
    class_sampling_frequency = len(temp)
    mean_of_all_ious_per_class = np.mean(temp)
    filtered_scores.append((_class, mean_of_all_ious_per_class, class_sampling_frequency))
filtered_train_ious = pd.DataFrame(filtered_scores, columns=["classes", "mean_train_ious", "class_sampling_frequency"])
filtered_train_ious = filtered_train_ious.sort_values(by="mean_train_ious", ascending=False)
#classes in meta-train and meta-val are unique if their intersection is zero
set(filtered_train_ious.classes.unique()).intersection(set(filtered_val_ious.classes.unique()))
meta_val_loss = pd.read_pickle(os.path.join(experiment_root_path, cached_files[5]))
meta_train_loss = pd.read_pickle(os.path.join(experiment_root_path, cached_files[3]))
num_data_points = meta_val_loss.shape[0]
fig = plt.figure(figsize=(10, 8))
plt.title("meta_losses")
plt.xlabel("epochs")
plt.ylabel("loss")
plt.plot(np.arange(num_data_points), meta_val_loss.total_val_loss, label="meta_val_loss")
plt.plot(np.arange(num_data_points), meta_train_loss.total_val_loss, label="meta_train_loss")
plt.legend()
results_path = "leo_segmentation/data/models/experiment_15"
cached_files = os.listdir(results_path)
cached_files = {i:k for i,k in enumerate(cached_files)}
cached_files
result_df = pd.read_pickle(os.path.join(results_path, cached_files[4]))
fig = plt.figure(figsize=(10, 7))
for i in range(5):
    plt.plot(result_df.iloc[:,i], label=result_df.columns[i])
plt.legend()
train_results_df = pd.read_pickle(os.path.join(results_path, cached_files[2]))
train_results_df.fillna(0.0).max().sort_values(ascending=False)
counts_per_col={}
for i in range(len(train_results_df.columns)):
    if train_results_df.columns[i] == "episode":
        continue
    temp = train_results_df.iloc[:,i]
    temp = temp.dropna()
    counts_per_col[train_results_df.columns[i]] = len(temp)
pd.Series(counts_per_col).sort_values(ascending=False)
fig = plt.figure(figsize=(10, 7))
plt.plot(pd.read_pickle(os.path.join(results_path,
            cached_files[3]))["total_val_loss"], label="train")
plt.plot(pd.read_pickle(os.path.join(results_path,
            cached_files[5]))["total_val_loss"], label="val")
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14257515942909202, 'fraction_numerical': 0.018979653811114484, 'mean_word_length': 6.014909478168264, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '220339', 'license': 'MIT', 'n_tokens_mistral': 2540, 'n_tokens_neox': 2425, 'n_words': 388, 'path': 'run_notebook.ipynb', 'repo_name': 'Temiloluwa/leo-segmentation-srp-project', 'size': '6586'}