<jupyter_start><jupyter_text>Chossing different classes<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt

def get_top_x_not_weighted(x):
    matrix = np.zeros(shape = (81, 81))
    weights = [1, 1, 1, 1, 1]
    for idx, neighbours in enumerate(classes_idx):
        neighbours = neighbours[:x+1]
        for position, neighbour in enumerate(neighbours[1:]):
            #if neighbour!=classes[idx]:
            matrix[int(classes[idx]), int(neighbour)] += weights[position] 
    for idx in range(matrix.shape[0]):
        matrix[idx] = np.round(matrix[idx]/ np.sum(matrix[idx], axis=0),4)
    return matrix

def get_df(matrix):
    import pandas as pd
    df = pd.DataFrame(matrix)
    names = ["background"] + [coco.cats[coco_cat]["name"] for (cat, coco_cat) in continious_cat_to_coco.items()]
    df.set_axis(names, axis=0)
    df.set_axis(names, axis=1)
    df.style.background_gradient()
    return df

def draw_df(df, name, figsize=(20, 20), ann=False ):
    fig, ax = plt.subplots(figsize=figsize)
    pal = sns.cubehelix_palette(light=1, as_cmap=True)

    sns_plot = sns.heatmap(ax = ax, data=df, cmap=pal, annot=ann)
    sns_plot.get_figure().savefig("nn_review/" + name)

m_5 = get_top_x_not_weighted(5)
d_5 = get_df(m_5)
draw_df(d, "top5.png")
m = get_top_x_not_weighted(1)
d = get_df(m)
draw_df(d, "top1.png")
m_5[1], m[1]
df.to_pickle("nn_review/nn_to_wrong_classes_5.pkl")
df.style.background_gradient().to_excel("nn_review/nn_to_wrong_classes_5.xlsx")<jupyter_output><empty_output><jupyter_text>Chosing yourself<jupyter_code>matrix = np.zeros(shape = (81,4))
for idx, neighbours in enumerate(classes_idx):
    for position, neighbour in enumerate(neighbours[1:]):
        if neighbour==classes[idx]:
            matrix[int(neighbour)][position]+=1
for idx in range(matrix.shape[0]):
    matrix[idx] = np.round(matrix[idx]/ np.sum(matrix[idx]),3)*100
np.sum(matrix[5])
import pandas as pd
df = pd.DataFrame(matrix)
names = ["background"] + [coco.cats[coco_cat]["name"] for (cat, coco_cat) in continious_cat_to_coco.items()]
df.set_axis(names, axis=0)
df.style.background_gradient()
draw_df(df, "same_class_%", figsize=(5,20), ann=True)
df.to_pickle("nn_review/class_place_apperance.pkl")
df.style.background_gradient().to_excel("nn_review/class_place_apperance.xlsx")<jupyter_output><empty_output><jupyter_text>Mean distance<jupyter_code>names = [coco.cats[continious_cat_to_coco[x]]["name"] for x in average_distance_class.keys()]
import matplotlib.pyplot as plt
#plt.figure(figsize=(20, 20))

#plt.bar(average_distance_class.keys(), average_distance_class.values())
dictionary = {"class": [], "mean_distance": []}
for key,value in average_distance_class.items():
    dictionary["class"].append(coco.cats[continious_cat_to_coco[key]]["name"])
    dictionary["mean_distance"].append(value)
df = pd.DataFrame(dictionary)
df = df.sort_values('mean_distance')
ax = df.plot.barh(x='class', y='mean_distance', rot=0, figsize=(20,20))
fig = ax.get_figure()

fig.savefig("nn_review/median_distance.png")

#x = list(range(81))
#my_xticks = [coco.cats[coco_cat]["name"] for (cat, coco_cat) in continious_cat_to_coco.items()]
#plt.xticks(x, my_xticks, rotation=45)

#plt.show()<jupyter_output><empty_output><jupyter_text>Distance to different classes<jupyter_code>tmp_dict= {}
for idx in range(len(distances)):
    me = int(classes[idx])
    for distance,other in zip(distances[idx], classes_idx[idx]):
        key = (me, int(other))
        if key not in tmp_dict.keys():
            tmp_dict[key] = []
        if distance<0:
            distance=0
        tmp_dict[key].append(distance)    
matrix = np.zeros((81, 81))
for (me, other) in tmp_dict.keys():
    matrix[me][other] = np.median(tmp_dict[(me, other)])
import pandas as pd
df = pd.DataFrame(matrix)
names = ["background"] + [coco.cats[coco_cat]["name"] for (cat, coco_cat) in continious_cat_to_coco.items()]
df.set_axis(names, axis=0)
df.set_axis(names, axis=1)
df.style.background_gradient()
draw_df(df, name="median_distance", ann=False)
df.to_pickle("nn_review/mean_distance_to_different_classes.pkl")
df.style.background_gradient().to_excel("nn_review/mean_distance_to_different_classes.xlsx")
def choose_gradient_id_for_background(faiss_db, feautre_db, looked_features, thresholds, k_neighbours ):
    distance, indecies = faiss_db.search(looked_features, k_neighbours)
    #feautre_db[indecies].shape -> number of features, k_neighbours, 1031 dimension vector
    classes = feautre_db[indecies][:, :, 2]
    max_classes = [np.argmax(np.bincount(sample.astype("int64"))) for sample in classes]
    class_thresholds = thresholds[max_classes]
    passed_thresholds = np.where( np.min(distance, axis=1) < class_thresholds)[0]
    background_gradient_idx = np.array(set(range(len(looked_features))) - set(passed_thresholds))
    return background_gradient_idx
background_gradient_idx = choose_gradient_id_for_background(faiss_db, feautre_db, looked_features, thresholds, k_neighbours )

background_gradient_idx = choose_gradient_id_for_background(faiss_db, feautre_db, looked_features, thresholds, k_neighbours )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14494732735076082, 'fraction_numerical': 0.015216543113538821, 'mean_word_length': 5.919028340080971, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '16214', 'license': 'MIT', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1892, 'n_words': 317, 'path': 'notebooks/Thresholds.ipynb', 'repo_name': 'elnazavr/Detectron.pytorch', 'size': '5126'}