<jupyter_start><jupyter_text>Data Gen<jupyter_code>def softmax(x):
    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)

class DataGen:
    """
    A data generator that can handing multiple workers if used in conjunction with
    the model.fit_generator method.
    Enables simplification of images by using pixelation e.g. 128x128 >> 16x16 >> 128x128,
    paired with smooth transitions between scales using alpha, as well as gaussian blurring
    controlled by sigma.
    It is a generator so `X, y = next(generator)` will get a new batch, but indexing also works and
    is preferred e.g. `X, y = generator[0]`.
    """

    def __init__(self, img_dir, batch_size, shuffle=True, apply_to_all=False):
        self.img_dir = img_dir
        self.batch_size = batch_size
        self.files = np.array(os.listdir(self.img_dir))
        self.n = len(self.files)
        self.indices = np.arange(0, self.n)
        if shuffle:
            np.random.shuffle(self.indices)
        self.pos = 0
        self.apply_to_all = apply_to_all

    def _get_indices(self, i):
        return self.indices[i * self.batch_size:i * self.batch_size + self.batch_size]

    def __getitem__(self, i):
        batch = []
        labels = []
        files = self.files[self._get_indices(i)]
        for file in files:
            # Get input
            #img = one_hot(np.expand_dims(np.argmax(np.load(os.path.join(self.img_dir, file)), axis=-1), 0))
            img = np.expand_dims(np.load(os.path.join(self.img_dir, file)), 0)
            
            # Get label
            if "HEAL" in file:
                labels.append(np.array([1, 0, 0, 0.]))
            elif "BCC" in file:
                labels.append(np.array([0., 1, 0, 0]))
            elif "IEC" in file:
                labels.append(np.array([0, 0., 1, 0]))
            elif "SCC" in file:
                labels.append(np.array([0, 0, 0., 1]))
                
            if self.apply_to_all:
                img = softmax(img)
            # Add to batch
            batch.append(img)
                    
        return batch[0], np.stack(labels)

    def __next__(self):
        self.pos += 1
        if self.pos > self.n // self.batch_size:
            self.pos = 1
        return self.__getitem__(self.pos - 1)
    
    def set_files(self, files):
        self.files = np.array([os.path.join(self.img_dir, file) for file in files])
        self.n = len(self.files)
        self.indices = np.arange(0, self.n)
        np.random.shuffle(self.indices)
        self.pos = 0
        print("files updated")
train_gen = DataGen(DATA_DIR, 1, apply_to_all=True)
val_gen = DataGen(DATA_DIR, 1, apply_to_all=True)
test_gen = DataGen(DATA_DIR, 1, apply_to_all=True)
train_gen.set_files(train_files)
val_gen.set_files(val_files)
test_gen.set_files(test_files)<jupyter_output>files updated
files updated
files updated<jupyter_text>Model<jupyter_code>from keras.models import Model
from keras.layers import *
from keras.optimizers import Adam
import keras.backend as K
K.clear_session()

model_in = Input(shape=(None, None, 12))
x = Conv2D(32, (3, 3), padding="same", activation="relu")(model_in)
x = Conv2D(32, (3, 3), padding="same", activation="relu")(x)
x = Conv2D(32, (3, 3), padding="same", activation="relu")(x)
x = GlobalAveragePooling2D()(x)
x = Dense(16, activation="relu")(x)
model_out = Dense(4, activation="softmax")(x)
model = Model(inputs=[model_in], outputs=[model_out])
model.summary()
model.compile(optimizer=(Adam(lr=0.001)), loss='categorical_crossentropy', metrics=["acc"])
# history = model.fit_generator(generator=train_gen,
#                               steps_per_epoch=train_gen.n,
#                               epochs=1,
#                               validation_data=val_gen,
#                               validation_steps=val_gen.n,
#                               workers=6,
#                               use_multiprocessing=True,
#                               #callbacks=[check_point]
#                              )
#model.save_weights("./classification_weights__EPOCH_7_train_95_val_1.h5")
model.load_weights("./classification_weights__EPOCH_5_train_95_val_1.h5")<jupyter_output><empty_output><jupyter_text><jupyter_code>loss, acc = model.evaluate_generator(generator=test_gen, steps=test_gen.n, verbose=1)
acc # epoch 4
loss, acc = model.evaluate_generator(generator=test_gen, steps=test_gen.n, verbose=1)
acc #  EPOCH 5
true = []
pred = []
for i in range(test_gen.n):
    X, y = test_gen[i]
    
    
    y_pred = model.predict(X)
    
    pred.append(y_pred)
    true.append(y)
    
true = np.vstack(true)
pred = np.vstack(pred)
pred.shape
np.argmax(pred, axis=-1) == np.argmax(true, axis=-1).flatten()
cm = np.zeros((4, 4))
y_pred = np.argmax(pred, axis=-1)
y_true = np.argmax(true, axis=-1).flatten()

np.sum(y_pred == y_true) / len(y_true)
for i in range(len(y_pred)):
    r = y_true[i]
    c = y_pred[i]
    cm[r, c] += 1
cm<jupyter_output><empty_output><jupyter_text>Check distributions<jupyter_code># TRAIN
distros = []
n = len(train_gen.files)
for i, file in enumerate(train_gen.files):
    print(i, "of", n)
    img = np.load(file)
    dist = np.mean(img, axis=(0, 1))
    distros.append(dist)
distros = np.stack(distros)
# Validation
val_distros = []
n = len(val_gen.files)
for i, file in enumerate(val_gen.files):
    print(i, "of", n)
    img = np.load(file)
    dist = np.mean(img, axis=(0, 1))
    val_distros.append(dist)
val_distros = np.stack(val_distros)
# Test
test_distros = []
n = len(test_gen.files)
for i, file in enumerate(test_gen.files):
    print(i, "of", n)
    img = np.load(file)
    dist = np.mean(img, axis=(0, 1))
    test_distros.append(dist)
test_distros = np.stack(test_distros)
# ------------ EXTERNAL --------------------------------------- #
EXT_DATA_DIR = "/media/simon/UNTITLED/PROB_MAPS_REDO/maps/"
ext_gen = DataGen(EXT_DATA_DIR, 1, apply_to_all=True)

ext_gen.files = [os.path.join(EXT_DATA_DIR, file) for file in ext_gen.files]

# Test
ext_distros = []
n = len(ext_gen.files)
for i, file in enumerate(ext_gen.files):
    print(i, "of", n)
    img = np.load(file)
    dist = np.mean(img, axis=(0, 1))
    ext_distros.append(dist)
ext_distros = np.stack(ext_distros)<jupyter_output>0 of 116
1 of 116
2 of 116
3 of 116
4 of 116
5 of 116
6 of 116
7 of 116
8 of 116
9 of 116
10 of 116
11 of 116
12 of 116
13 of 116
14 of 116
15 of 116
16 of 116
17 of 116
18 of 116
19 of 116
20 of 116
21 of 116
22 of 116
23 of 116
24 of 116
25 of 116
26 of 116
27 of 116
28 of 116
29 of 116
30 of 116
31 of 116
32 of 116
33 of 116
34 of 116
35 of 116
36 of 116
37 of 116
38 of 116
39 of 116
40 of 116
41 of 116
42 of 116
43 of 116
44 of 116
45 of 116
46 of 116
47 of 116
48 of 116
49 of 116
50 of 116
51 of 116
52 of 116
53 of 116
54 of 116
55 of 116
56 of 116
57 of 116
58 of 116
59 of 116
60 of 116
61 of 116
62 of 116
63 of 116
64 of 116
65 of 116
66 of 116
67 of 116
68 of 116
69 of 116
70 of 116
71 of 116
72 of 116
73 of 116
74 of 116
75 of 116
76 of 116
77 of 116
78 of 116
79 of 116
80 of 116
81 of 116
82 of 116
83 of 116
84 of 116
85 of 116
86 of 116
87 of 116
88 of 116
89 of 116
90 of 116
91 of 116
92 of 116
93 of 116
94 of 116
95 of 116
96 of 116
97 of 116
98 of 116
99 of 116
100 of 116[...]<jupyter_text>SHOW THEM<jupyter_code>for distro in [distros, val_distros, test_distros, ext_distros]:
    plt.title("")
    bplot = plt.boxplot(distro,
                    showfliers=False,
                    medianprops=dict(color="black"),
                    patch_artist=True
                   )

    for patch, color in zip(bplot["boxes"], colors):
        patch.set_facecolor(color)

    plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
    plt.show()<jupyter_output><empty_output><jupyter_text>Other data<jupyter_code>loss, acc = model.evaluate_generator(generator=ext_gen, steps=ext_gen.n, verbose=1)
acc
true = []
pred = []
for i in range(ext_gen.n):
    
    print(i, "of", ext_gen.n)
    X, y = ext_gen[i]
    
    
    y_pred = model.predict(X)
    
    pred.append(y_pred)
    true.append(y)
    
true = np.vstack(true)
pred = np.vstack(pred)
cm = np.zeros((4, 4))
y_pred = np.argmax(pred, axis=-1)
y_true = np.argmax(true, axis=-1).flatten()

np.sum(y_pred == y_true) / len(y_true)
for i in range(len(y_pred)):
    r = y_true[i]
    c = y_pred[i]
    cm[r, c] += 1
cm<jupyter_output><empty_output><jupyter_text>Random Forrest<jupyter_code>X_train_f = []
y_train_f = []

for j in range(train_gen.n):
    
    print(j, "of",train_gen.n )
    X, y = train_gen[j]
    
    groups, counts = np.unique(np.argmax(X, axis=-1), return_counts=True)
    total = np.sum(counts)
    vector = np.zeros(12)
    for i,g in enumerate(groups):
        vector[g] = counts[i] / total
        
        
    X_train_f.append(vector)
    y_train_f.append(np.argmax(y, axis=-1))
    
X_train_f = np.vstack(X_train_f)
X_val_f = []
y_val_f = []

for j in range(val_gen.n):
    
    print(j, "of",val_gen.n )
    X, y = val_gen[j]
    
    groups, counts = np.unique(np.argmax(X, axis=-1), return_counts=True)
    total = np.sum(counts)
    vector = np.zeros(12)
    for i,g in enumerate(groups):
        vector[g] = counts[i] / total
        
        
    X_val_f.append(vector)
    y_val_f.append(np.argmax(y, axis=-1))
    
X_val_f = np.vstack(X_val_f)
X_test_f = []
y_test_f = []

for j in range(test_gen.n):
    
    print(j, "of",test_gen.n )
    X, y = test_gen[j]
    
    groups, counts = np.unique(np.argmax(X, axis=-1), return_counts=True)
    total = np.sum(counts)
    vector = np.zeros(12)
    for i,g in enumerate(groups):
        vector[g] = counts[i] / total
        
        
    X_test_f.append(vector)
    y_test_f.append(np.argmax(y, axis=-1))
    
X_test_f = np.vstack(X_test_f)<jupyter_output>0 of 47
1 of 47
2 of 47
3 of 47
4 of 47
5 of 47
6 of 47
7 of 47
8 of 47
9 of 47
10 of 47
11 of 47
12 of 47
13 of 47
14 of 47
15 of 47
16 of 47
17 of 47
18 of 47
19 of 47
20 of 47
21 of 47
22 of 47
23 of 47
24 of 47
25 of 47
26 of 47
27 of 47
28 of 47
29 of 47
30 of 47
31 of 47
32 of 47
33 of 47
34 of 47
35 of 47
36 of 47
37 of 47
38 of 47
39 of 47
40 of 47
41 of 47
42 of 47
43 of 47
44 of 47
45 of 47
46 of 47<jupyter_text><jupyter_code>from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(max_depth=2, random_state=0, n_estimators=100)
clf.fit(X_train_f, y_train_f)
clf.score(X_train_f, y_train_f)
clf.score(X_val_f, y_val_f)
clf.score(X_test_f, y_test_f)
y_pred = clf.predict(X_test_f)
y_true = np.array(y_test_f).flatten()
cm = np.zeros((4, 4))

for i in range(len(y_pred)):
    r = y_true[i]
    c = y_pred[i]
    cm[r, c] += 1

print(cm)
features = clf.feature_importances_<jupyter_output><empty_output><jupyter_text>Plot Feature Importance<jupyter_code>ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)

# Points
plt.bar(np.arange(12), features, color=colors, zorder=10)

plt.xticks(ticks=range(0, 12), labels=[color_dict[_][0] for _ in range(12)])
#plt.title("Two Classes")
plt.ylabel("Feature Importance")
plt.savefig("./feature_importance-random_forrest.png", dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Average Training<jupyter_code>X_train_f.shape
def encoder(pos):
    result = np.zeros(4)
    result[pos] = 1.
    return result
y_train = np.stack([encoder(pos[0]) for pos in y_train_f])
y_val = np.stack([encoder(pos[0]) for pos in y_val_f])<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>K.clear_session()

model_in = Input(shape=(12,))
x = Dense(64, activation="relu")(model_in)
model_out = Dense(4, activation="softmax")(x)
avg_model = Model(inputs=[model_in], outputs=[model_out])
avg_model.summary()
avg_model.compile(optimizer=(Adam(lr=0.001)), loss='categorical_crossentropy', metrics=["acc"])
history = avg_model.fit(x=X_train_f, y=y_train, batch_size=1, epochs=10,
                    validation_data=[X_val_f, y_val])
y_test = np.stack([encoder(pos[0]) for pos in y_test_f])
pred = avg_model.predict(X_test_f)
result = np.argmax(pred, axis=-1) == np.argmax(y_test, axis=-1)
np.sum(result) / len(result)
y_true = np.argmax(y_test, axis=-1)
y_pred = np.argmax(pred, axis=-1)
cm = np.zeros((4, 4))

for i in range(len(y_pred)):
    r = y_true[i]
    c = y_pred[i]
    cm[r, c] += 1

print(cm)<jupyter_output>[[13.  3.  0.  0.]
 [ 2. 13.  1.  0.]
 [ 0.  1.  6.  0.]
 [ 0.  0.  0.  8.]]<jupyter_text>Nearest NeighBors<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
X_train_f.shape
neigh = KNeighborsClassifier(n_neighbors=4)
neigh.fit(X_train_f, y_train_f)
neigh.score(X_train_f, y_train_f)
neigh.score(X_val_f, y_val_f)
neigh.score(X_test_f, y_test_f)
y_pred = neigh.predict(X_test_f)
y_true = np.array([ thing[0] for thing in y_test_f])
cm = np.zeros((4, 4))

for i in range(len(y_pred)):
    r = y_true[i]
    c = y_pred[i]
    cm[r, c] += 1

print(cm)<jupyter_output>[[11.  1.  4.  0.]
 [ 0. 14.  2.  0.]
 [ 2.  0.  5.  0.]
 [ 0.  0.  0.  8.]]<jupyter_text>Gradients<jupyter_code>def reset_weights(model):
    import keras.backend as K
    session = K.get_session()
    for layer in model.layers: 
        if hasattr(layer, 'kernel_initializer'): 
            layer.kernel.initializer.run(session=session)
        if hasattr(layer, 'bias_initializer'):
            layer.bias.initializer.run(session=session)
reset_weights(model)
all_files = train_gen.files
healthy = [ file for file in all_files if "HEAL" in file]
bcc = [ file for file in all_files if "BCC" in file]
iec = [ file for file in all_files if "IEC" in file]
scc = [ file for file in all_files if "SCC" in file]
def normalize(x):
    """utility function to normalize a tensor.

    # Arguments
        x: An input tensor.

    # Returns
        The normalized input tensor.
    """
    return x / (K.sqrt(K.mean(K.square(x))) + K.epsilon())
gen = DataGen(".", 1, apply_to_all=True)
X, y = gen[3]
plt.imshow(np.argmax(X, axis=-1)[0], vmin=0, vmax=11, cmap=seg_cmap)
plt.show()
gen.set_files(healthy)
pos = 0

loss = -model.layers[-1].output[:, pos]
input_tensor = model.inputs[0]
grads = K.gradients(loss, input_tensor)[0]
grads = normalize(grads)
# this function returns the loss and grads given the input picture
get_grads = K.function([input_tensor], [grads])


healthy_features = []
for i in range(gen.n):
    
    print(i+1, "of", gen.n)
    
    X, y = gen[i]
    
    g = get_grads([X])[0][0]
    
    features = np.mean(np.abs(g), axis=(0, 1))
    
    features /= features.max()
    
    healthy_features.append(features)

#     fig, ax = plt.subplots(1, 2, figsize=(12, 12))
#     ax[0].imshow(np.argmax(X, axis=-1)[0], vmin=0, vmax=11, cmap=seg_cmap)
#     #plt.show()
    
#     img = np.mean(np.abs(g), axis=-1)
#     img /= img.max()
    
#     ax[1].imshow(img, cmap="magma")
#     plt.show()

healthy_features = np.stack(healthy_features)
test_gen.files[0].split("/")[-1].split(".")[0] + ".png"
gen.set_files(test_gen.files)


for i in range(gen.n):
    
    print(i+1, "of", gen.n)
    
    X, y = gen[i]
    
    
    pos = np.argmax(y)
    
    # Get Grads
    loss = -model.layers[-1].output[:, pos]
    input_tensor = model.inputs[0]
    grads = K.gradients(loss, input_tensor)[0]
    grads = normalize(grads)
    # this function returns the loss and grads given the input picture
    get_grads = K.function([input_tensor], [grads])
    g = get_grads([X])[0][0]


    fig, ax = plt.subplots(1, 2, figsize=(12, 12))
    ax[0].imshow(np.argmax(X, axis=-1)[0], vmin=0, vmax=11, cmap=seg_cmap)
    ax[0].axis("off")
    #plt.show()
    
    img = np.mean(np.abs(g), axis=-1)
    img /= img.max()
    
    ax[1].imshow(img, cmap="magma")
    ax[1].axis("off")
    
    
    plt.savefig(f"./FEATURE_IMPORTANCE/{i}.png", dpi=100)
    plt.close()
bplot = plt.boxplot(healthy_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)

plt.title('Classification of Healthy')

plt.savefig("./relative_feature_importance_Healthy.png", dpi=300)
plt.show()
bplot = plt.boxplot(bcc_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)

plt.title('Classification of BCC')

plt.savefig("./relative_feature_importance_BCC.png", dpi=300)
plt.show()
bplot = plt.boxplot(iec_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)

plt.title('Classification of IEC')

plt.savefig("./relative_feature_importance_IEC.png", dpi=300)
plt.show()
bplot = plt.boxplot(scc_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)

plt.title('Classification of SCC')

plt.savefig("./relative_feature_importance_SCC.png", dpi=300)
plt.show()
combined_features = np.vstack([healthy_features, bcc_features, iec_features, scc_features])
bplot = plt.boxplot(combined_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)


plt.savefig("./relative_feature_importance_ALL.png", dpi=300)
plt.show()
random_combined_features = np.vstack([healthy_features, bcc_features, iec_features, scc_features])
bplot = plt.boxplot(random_combined_features,
                showfliers=False,
                medianprops=dict(color="black"),
                patch_artist=True
               )
plt.xticks(ticks=range(1, 13), labels=[color_dict[_][0] for _ in range(12)])
plt.ylabel("Relative Feature Importance")

for patch, color in zip(bplot["boxes"], colors):
    patch.set_facecolor(color)
    
ax = plt.gca()

ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)


plt.savefig("./relative_feature_importance_ALL_RANDOM.png", dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>cm = np.array([[16.,  0.,  0.,  0.],
       [ 1., 14.,  1.,  0.],
       [ 0.,  1.,  6.,  0.],
       [ 0.,  0.,  0.,  8.]])
cm = cm / np.sum(cm, axis=1)<jupyter_output><empty_output><jupyter_text>![image.png](attachment:image.png)<jupyter_code>#from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from mpl_toolkits.axes_grid1 import make_axes_locatable
dark_purple = (162/255., 30/255., 140/255.) #rgb(64%, 12%, 55%)
grey = (0.94, 0.94, 0.94)

norm=plt.Normalize(0,1)
cmap = LinearSegmentedColormap.from_list("", [grey, dark_purple])
fig, ax = plt.subplots(1, 1, figsize=(6, 6))
cs = ax.matshow(cm, cmap=cmap)
ax.set_xlabel("Predict", fontsize=15)
ax.set_ylabel("True", fontsize=15)


ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.spines['left'].set_visible(False)


ax.set_xticklabels(["_", "Healthy", "BCC", "IEC", "SCC"])
ax.set_yticklabels(["_", "Healthy", "BCC", "IEC", "SCC"])

ax.tick_params(length=0, which="both", labelsize=12)

ax.set_xticks([x - 0.5 for x in plt.gca().get_xticks()][1:], minor='true')
ax.set_yticks([y - 0.5 for y in plt.gca().get_yticks()][1:], minor='true')
ax.grid(which='minor', color="white", lw=3)

# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)
cb = plt.colorbar(cs, cax=cax)
cb.outline.set_visible(False)
cb.ax.tick_params(length=0)

for r in range(4):
    for c in range(4):
        num = np.round(cm[r, c], 3)
        
        if num == 0:
            continue
            
        if r == c:
            if num == 1:
                ax.text(c-0.12, r+0.05, str(num), color="white", fontsize=15)
            else:
                ax.text(c-0.25, r+0.05, str(num), color="white", fontsize=15)
        
        else:
            ax.text(c-0.25, r+0.05, str(num), color="black", fontsize=15)
plt.tight_layout()
plt.savefig("./confusion_matrix_TEST.png", dpi=300)         
plt.show()
# binary
np.sum(cm[1:, 1:])
cm
binary = np.array([[16, 0],
                   [1, 30]])
np.sum(np.diag(binary)) / np.sum(cm)
# RF

cm = np.array([[13.,  2.,  1.,  0.],
                 [ 1., 14.,  1.,  0.],
                 [ 0.,  1.,  6.,  0.],
                 [ 0.,  0.,  0.,  8.]])


binary = np.array([[16, 3],
                   [1, np.sum(cm[1:, 1:])]])

np.sum(np.diag(binary)) / np.sum(cm)


# KNN
cm = np.array([  [11.,  1.,  4.,  0.],
                 [ 0., 14.,  2.,  0.],
                 [ 2.,  0.,  5.,  0.],
                 [ 0.,  0.,  0.,  8.]])


binary = np.array([[11, 5],
                   [2, np.sum(cm[1:, 1:])]])

np.sum(np.diag(binary)) / np.sum(cm)

# SLP

cm = np.array([[13.,  3.,  0.,  0.],
               [ 2., 13.,  1.,  0.],
               [ 0.,  1.,  6.,  0.],
               [ 0.,  0.,  0.,  8.]])

binary = np.array([[13, 3],
                   [2, np.sum(cm[1:, 1:])]])

np.sum(np.diag(binary)) / np.sum(cm)<jupyter_output><empty_output><jupyter_text>External Data<jupyter_code>cm = np.array([ [40.,  0.,  0.,  0.],
                [ 2., 22.,  3.,  0.],
                [ 1.,  1., 21.,  1.],
                [ 0.,  0.,  6., 19.]])
cm = cm / np.sum(cm, axis=1)
fig, ax = plt.subplots(1, 1, figsize=(6, 6))
cs = ax.matshow(cm, cmap=cmap)
ax.set_xlabel("Predict", fontsize=15)
ax.set_ylabel("True", fontsize=15)


ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.spines['left'].set_visible(False)


ax.set_xticklabels(["_", "Healthy", "BCC", "IEC", "SCC"])
ax.set_yticklabels(["_", "Healthy", "BCC", "IEC", "SCC"])

ax.tick_params(length=0, which="both", labelsize=12)

ax.set_xticks([x - 0.5 for x in plt.gca().get_xticks()][1:], minor='true')
ax.set_yticks([y - 0.5 for y in plt.gca().get_yticks()][1:], minor='true')
ax.grid(which='minor', color="white", lw=3)

# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)
cb = plt.colorbar(cs, cax=cax)
cb.outline.set_visible(False)
cb.ax.tick_params(length=0)

for r in range(4):
    for c in range(4):
        num = np.round(cm[r, c], 3)
        
        if num == 0:
            continue
            
        if r == c:
            if num == 1:
                ax.text(c-0.12, r+0.05, str(num), color="white", fontsize=15)
            else:
                ax.text(c-0.25, r+0.05, str(num), color="white", fontsize=15)
        
        else:
            ax.text(c-0.25, r+0.05, str(num), color="black", fontsize=15)
plt.tight_layout()
plt.savefig("./confusion_matrix_EXT.png", dpi=300)         
plt.show()
np.sum(cm[1:, 1:])
cm = np.array([[40.,  0.,  0.,  0.],
       [ 2., 22.,  3.,  0.],
       [ 1.,  1., 21.,  1.],
       [ 0.,  0.,  6., 19.]])
import numpy as np
binary = np.array([[40, 0],
                   [3, 73]])

np.sum(np.diag(binary)) / np.sum(binary)
np.sum(np.diag(cm)) / np.sum(cm)
np.sum(cm)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15712600187151635, 'fraction_numerical': 0.05289067903494853, 'mean_word_length': 2.934688650552265, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '617493', 'license': 'Apache-2.0', 'n_tokens_mistral': 10517, 'n_tokens_neox': 9555, 'n_words': 2155, 'path': '10_classification.ipynb', 'repo_name': 'smthomas-sci/SkinCancerSegmentation', 'size': '24579'}