<jupyter_start><jupyter_text>Helper function to plot<jupyter_code>def plot_graph(axis_title, x, y_train, y_val, xlabel, ylabel, xtick_range, ytick_range, save_path=None):

    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(9, 6))

    line1 = ax.plot(x, y_train, color="blue", label="train")
    line2 = ax.plot(x, y_val, color="red", label="val")

    # Nicer visuals.
    ax.set_title(axis_title)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.grid(b=True, which="major", axis="both", color="#d3d3d3", linestyle="-")
    ax.grid(b=True, which="minor", axis="both", color="#e7e7e7", linestyle="dashed")
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.xaxis.set_ticks(np.arange(xtick_range[0], xtick_range[1], xtick_range[2]))
    ax.xaxis.set_minor_locator(MultipleLocator(5))
    ax.set_xlim(left=0)
    ax.yaxis.set_ticks(np.arange(ytick_range[0], ytick_range[1], ytick_range[2]))
    ax.yaxis.set_minor_locator(MultipleLocator(0.01))
    ax.patch.set_alpha(0)
    
    handles, labels = ax.get_legend_handles_labels()
    ax.legend(handles=handles, labels=labels, loc="best")
    
    # Save graph
    plt.tight_layout()
    if save_path:
        plt.savefig(fname=save_path, dpi=300)<jupyter_output><empty_output><jupyter_text>Batch-size = 64<jupyter_code>df_64 = pd.read_csv("../results/fit/20201203_040325___INSTANCE/csv_logger/csv_logger.csv")
df_64.head()
plot_graph(axis_title="Batch Size = 64",
           x=df_64["epoch"],
           y_train=df_64["loss"],
           y_val=df_64["val_loss"],
           xlabel="Epoch",
           ylabel="Binary crossentropy loss",
           xtick_range=(0, 50, 10),
           ytick_range=(0, 0.16, 0.02),
           save_path="../results/fit/20201203_040325___INSTANCE/csv_logger/loss.png")
plot_graph(axis_title="Batch Size = 64",
           x=df_64["epoch"],
           y_train=df_64["iouMetric"],
           y_val=df_64["val_iouMetric"],
           xlabel="Epoch",
           ylabel="iouMetric",
           xtick_range=(0, 50, 10),
           ytick_range=(0, 0.26, 0.02),
           save_path="../results/fit/20201203_040325___INSTANCE/csv_logger/iouMetric.png")<jupyter_output><empty_output><jupyter_text>Batch-size = 10<jupyter_code>df_10 = pd.read_csv("../results/fit/20201203_013807___INSTANCE/csv_logger/csv_logger.csv")
df_10.head()
plot_graph(axis_title="Batch Size = 10",
           x=df_10["epoch"],
           y_train=df_10["loss"],
           y_val=df_10["val_loss"],
           xlabel="Epoch",
           ylabel="Binary crossentropy loss",
           xtick_range=(0, 50, 10),
           ytick_range=(0, 0.16, 0.02),
           save_path="../results/fit/20201203_013807___INSTANCE/csv_logger/loss.png")
plot_graph(axis_title="Batch Size = 10",
           x=df_10["epoch"],
           y_train=df_10["iouMetric"],
           y_val=df_10["val_iouMetric"],
           xlabel="Epoch",
           ylabel="iouMetric",
           xtick_range=(0, 50, 10),
           ytick_range=(0, 0.26, 0.02),
           save_path="../results/fit/20201203_013807___INSTANCE/csv_logger/iouMetric.png")<jupyter_output><empty_output><jupyter_text>--- Combine .csv<jupyter_code>mass_train_df = pd.read_csv("../data/raw_data/csv-description-updated/Mass-Training-Description-UPDATED.csv")
mass_test_df = pd.read_csv("../data/raw_data/csv-description-updated/Mass-Test-Description-UPDATED.csv")

mass_df = pd.concat([mass_train_df, mass_test_df])

mass_df
# Create identifier column.
mass_df.insert(loc=0, column="identifier", value=np.nan)
mass_df["identifier"] = mass_df.apply(lambda x: "_".join([x["patient_id"], x["left_or_right_breast"], x["image_view"]]), axis=1)

# Drop filepath columns, they are useless because the filepaths always change.
mass_df.drop(["image_file_path", "cropped_image_file_path", "ROI_mask_file_path", "full_path", "mask_path", "crop_path"], axis=1, inplace=True)

# Sort by identifier column.
mass_df.sort_values(by=["identifier"])
mass_df.to_csv("../data/csv/Mass_all.csv", index=False)
# Get list of test and train image identifiers.
train_identifiers = []
test_identifiers = []

train_path = "../data/preprocessed/Mass/Train_FULL"
test_path = "../data/preprocessed/Mass/Test_FULL"

# Train images.
for curdir, dirs, files in os.walk(train_path):
    
    files.sort()
    
    for f in files:
        if f.endswith(".png"):
            f = f.replace("_FULL___PRE.png", "")
            train_identifiers.append(f)
            
# Test images.
for curdir, dirs, files in os.walk(test_path):
    
    files.sort()
    
    for f in files:
        if f.endswith(".png"):
            f = f.replace("_FULL___PRE.png", "")
            test_identifiers.append(f)
            
print(len(train_identifiers))
print(train_identifiers[:5])
print(len(test_identifiers))
print(test_identifiers[:5])

# Create dataframe for train images.
mass_train_df_new = mass_df[mass_df["identifier"].isin(train_identifiers)]

# Create dataframe for test images.
mass_test_df_new = mass_df[mass_df["identifier"].isin(test_identifiers)]
print(mass_train_df_new.shape)
print(mass_test_df_new.shape)
mass_train_df_new.to_csv("../data/csv/Mass_train.csv", index=False)
mass_test_df_new.to_csv("../data/csv/Mass_test.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15416825275980206, 'fraction_numerical': 0.0395888846593072, 'mean_word_length': 3.8477859778597785, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '667725', 'license': 'MIT', 'n_tokens_mistral': 2060, 'n_tokens_neox': 1932, 'n_words': 261, 'path': 'notebooks/plotTrainingGraphs.ipynb', 'repo_name': 'CleonWong/Can-You-Find-The-Tumour', 'size': '5254'}