<jupyter_start><jupyter_text>Loss Keys:  - B: Box Localization Loss   - C: Class Confidence Loss   - M: Mask Loss   - P: Prototype Loss   - D: Coefficient Diversity Loss   - E: Class Existence Loss   - S: Semantic Segmentation Loss<jupyter_code>import pandas as pd
nw_fdf = pd.DataFrame(nw_f)
nw_fdf = nw_fdf[nw_fdf["type"]=="train"]
d=list(nw_fdf["session"])
tot_ls = {}
# for u in range(d[-1]):
for u in nw_fdf["session"].unique()
    tot_ls[u] = (nw_fdf[nw_fdf["session"]==u])
# Extraxt the data that contains the losses
dt_dic = {}
# print(tot_ls[5])
dt_dic_df = {}
for we in tot_ls.keys():
    dt_dic[we] = []
    for a in range(len(tot_ls[we])):
        dt_dic[we].append(tot_ls[we].iloc[a,2])
    dt_dic_df[we] = pd.DataFrame(dt_dic[we])
# print(dt_dic[27])
#Generate the dataframe of the different losses
dt_dic2  = {}import pandas as pd
nw_fdf = pd.DataFrame(nw_fs)
nw_fdf = nw_fdf[nw_fdf["type"]=="train"]
d=list(nw_fdf["session"])
tot_ls = {}
# for u in range(d[-1]):
for u in nw_fdf["session"].unique()
    tot_ls[u] = (nw_fdf[nw_fdf["session"]==u])
# Extraxt the data that contains the losses
dt_dic = {}
# print(tot_ls[5])
dt_dic_df = {}
for we in tot_ls.keys():
    dt_dic[we] = []
    for a in range(len(tot_ls[we])):
        dt_dic[we].append(tot_ls[we].iloc[a,2])
    dt_dic_df[we] = pd.DataFrame(dt_dic[we])
# print(dt_dic[27])
#Generate the dataframe of the different losses
dt_dic2  = {}
dt_dic2_df = {}
for p in dt_dic_df.keys():
    dt_dic2[p] = []
    if len(dt_dic_df[p]) > 1:
        for n in range(len(dt_dic_df[p])):
            dt_dic2[p].append(dt_dic_df[p].iloc[n,0])
        dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
    else:
        del dt_dic2[p]
# dt_dic2_df[2]


nw_fdf["session"].unique()
dt_dic2_df = {}
for p in dt_dic_df.keys():
    dt_dic2[p] = []
    if len(dt_dic_df[p]) > 1:
        for n in range(len(dt_dic_df[p])):
            dt_dic2[p].append(dt_dic_df[p].iloc[n,0])
        dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
    else:
        del dt_dic2[p]
# dt_dic2_df[2]


nw_fdf["session"].unique()
from ast import literal_eval
import json 
import pandas as pd
import matplotlib.pyplot as plt
import tkinter as tk
import os
import time
from pathlib import Path
from tkinter import filedialog, simpledialog

def uichoosefile(title = None, initialdir = None):
    root = tk.Tk()
    root.withdraw() # we don't want a full GUI, so keep the root window from appearing
    filename = tk.filedialog.askopenfilename(title=title, initialdir = initialdir)
    return filename
def loss_values_analysis(file):
    with open(file) as f:
        f =  f.readlines()
    nw_f = []
    # unstring the string dataset
    for n in f:
        res = json.loads(n) 
        nw_f.append(res)
    # convert the dictionary to dataframe
    nw_fdf = pd.DataFrame(nw_f)
    nw_fdf = nw_fdf[nw_fdf["type"]=="train"]
    tot_ls = {}
    for u in nw_fdf["session"].unique():
        tot_ls[u] = (nw_fdf[nw_fdf["session"]==u])
    # Extraxt the data that contains the losses
    dt_dic = {}
    dt_dic_df = {}
    for we in tot_ls.keys():
        dt_dic[we] = []
        for a in range(len(tot_ls[we])):
            dt_dic[we].append(tot_ls[we].iloc[a,2])
        dt_dic_df[we] = pd.DataFrame(dt_dic[we])
    #Generate the dataframe of the different losses
    dt_dic2  = {}
    dt_dic2_df = {}
    for p in dt_dic_df.keys():
        dt_dic2[p] = []
        if len(dt_dic_df[p]) > 1:
            for n in range(len(dt_dic_df[p])):
                dt_dic2[p].append(dt_dic_df[p].iloc[n,0])
            dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
        else:
            del dt_dic2[p]
    return dt_dic2_df
# rt = loss_values_analysis(uichoosefile())
# rt = loss_values_analysis(file = "/home/basa/AR_yolact/yolact_base-old.log")
# rt[0] file:///home/basa/AR_yolact/logs/yolact_base_ExpCDC9.log
df = loss_values_analysis(file= "/home/basa/AR_yolact/logs/yolact_base_ExpCDC9.txt")
df[62]
df.keys()
print(len(df[59]['T']), len(df[62]['T']))
sess = 59
sf = df[59]['T']+df[62]['T']

six = df[62]['T']

df_list = list(df[59]['T'])
for i in range(len(df[62]['T'])):
    df_list.append(six[i])
# plt.plot(sf)
# print(sf)
df_list
# plt.plot(df_list)
# y_min = sf.min()
# y_max = df[sess]['T'].max()
# y = df[sess]['T']
# x_max = 0
# x_list = []
# for x in range(len(y)):
#     if y[x] == y_min:
#         x_list.append(x)
# y[0]
# # y[101299]
# # len(y)
# x_list
import numpy as np
df_mean = np.mean(df_list)
df_std = np.std(df_list)

x_upper = df_mean + 1.5*df_std
x_lower = df_mean - 1.5*df_std

plot_df  = []
for x in df_list:
    if x > x_upper or x < 0:
#         print(x)
        df_list.remove(x)
    else:
        plot_df.append(x)
        
plt.plot(plot_df)

y_min = np.min(plot_df)
y_max = df[sess]['T'].max()
y = plot_df
# x_max = 0
x_list = []
for x in range(len(y)):
    if y[x] == y_min:
        x_list.append(x)
print(x_list,y_min)
len(plot_df)
# df_mean
# x_upper
# x_lower
ax,fig = plt.figure()
fig = dt_dic2_df[2]["M"]
y_min = dt_dic2_df[2]["B"].min()
xpos = y.index(y_min)
x_min = x[xpos]
ax.annotate('local min', xy =(x_min,y_min), xytext = (x_min,y_min+5))
plt.show()
dt_dic2_df.keys()

# plt.plot(dt_dic2_df[2]["M"]
# dt_dic2_df[2]["M"]
# fig, ax = plt.subplots(2,2)
# plt.plot(dt_dic2_df[2]["M"])
# plt.annotate('min', xy = (0,0), xycoords ='data', xytext=(2,3),arrowprops= dict(facecolor='black', shrink=0.05))
# plt.show()
plt.semilogy(dt_dic2_df[32]["T"])
plt.show()
plt.plot(dt_dic2_df[32]["T"])
plt.show()
plt.plot(dt_dic2_df[2]["S"])
plt.plot(dt_dic2_df[2]["T"])
plt.plot(dt_dic2_df[21]["B"])
plt.plot(dt_dic2_df[21]["M"])
plt.plot(dt_dic2_df[21]["C"])
plt.plot(dt_dic2_df[21]["S"])
plt.plot(dt_dic2_df[19]["T"])
# import pandas as pd
# nw_fdf = pd.DataFrame(nw_f)
# nw_fdf=nw_fdf[nw_fdf["type"]=="train"]
# nw_fdf["session"]
# tot_ls = {}
# for u in nw_fdf["session"].unique():

#     tot_ls[u] = (nw_fdf[nw_fdf["session"]==u])
# dt_dic = {}
# dt_dic_df = {}
# # print(tot_ls.keys())
# for we in tot_ls.keys():
#     dt_dic[we] = []
# #     tot_df[we] = []
#     for a in range(len(tot_ls[we])):
#         if len(tot_ls[we])>1:
#             dt_dic[we].append(tot_ls[we].iloc[a,2])
# dt_dic_df = {}
# for yo in dt_dic.keys():
#         dt_dic_df[yo] = pd.DataFrame(dt_dic[yo])
        
# # dt_dic_df.keys()
# dt_dic2  = {}
# dt_dic2_df = {}
# for p in dt_dic_df.keys():
#     dt_dic2[p] = []
#     if "loss" in dt_dic_df[p].columns:
#         for n in range(len(dt_dic_df[p])):
#             dt_dic2[p].append(dt_dic_df[p].loc[n,"loss"])
#         dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
#     else:
#         del dt_dic2[p]
# dt_dic_df[9]
for g in range(5):
    print(int(g))
# import pandas as pd
# nw_fdf = pd.DataFrame(nw_f)

# nw_fdf["session"].unique()
# tot_ls = {}
# for u in nw_fdf["session"].unique():
# #     print(type(u))
# #     u = int(u)
#     tot_ls[u] = (nw_fdf[nw_fdf["session"]==u])
# # tot_ls
# dt_dic = {}
# print(tot_ls.keys())
# for we in tot_ls.keys():
# #     print(type(we))
#     dt_dic[we] = []
# #     tot_df[we] = []
#     for a in range(len(tot_ls[we])):
# #         if len(tot_ls[we])>1:
#         dt_dic[we].append(tot_ls[we].iloc[a,2])
# dt_dic_df = {}
# for yo in dt_dic.keys():
#         dt_dic_df[yo] = pd.DataFrame(dt_dic[yo])
# # type(dt_dic_df.keys())
# dt_dic2  = {}
# dt_dic2_df = {}
# for p in dt_dic_df.keys():
# #     print(type(p))
#     dt_dic2[p] = []
#     if "loss" in dt_dic_df[p].columns:
#         for n in range(len(dt_dic_df[p])):
#             dt_dic2[p].append(dt_dic_df[p].loc[n,"loss"])
#         dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
#     else:
#         del dt_dic2[p]
# # dt_dic2_df
# t_df = {}

# tt = {}
# for m in dt_dic2_df.keys():
# #     print(type(m))
#     tt_df[m] = []
#     for n in range(1, len(dt_dic2_df[m])):
#         tt_df[m].append(dt_dic2_df[m].iloc[n,0])
#     tt[m]= pd.DataFrame(tt_df[m])

import matplotlib.pyplot as plt
# plt.plot(tt[30]["T"])
tt
# dt_dic2  = {}
# dt_dic2_df = {}
# for p in dt_dic_df.keys():
#     dt_dic2[p] = []
#     for n in range(len(dt_dic_df[p])):
#         dt_dic2[p].append(dt_dic_df[p].iloc[n,-4])
#     dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
# dt_dic2_df[1]
dt_dic_df[9].iloc[3,-4]
#         dt_dic2[p].append(dt_dic_df[p].iloc[n,0])
#     dt_dic2_df[p]  = pd.DataFrame(dt_dic2[p])
# dt_dic2[27]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18894787644787644, 'fraction_numerical': 0.020632239382239383, 'mean_word_length': 3.4540569586243954, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '539552', 'license': 'MIT', 'n_tokens_mistral': 3973, 'n_tokens_neox': 3884, 'n_words': 636, 'path': 'Loss_Values_data.py.ipynb', 'repo_name': 'kogans1107/AR_yolact', 'size': '8288'}