<jupyter_start><jupyter_text>Aktives Lernen - Feedback für das GesamtmodellIm unüberwachten Szenario ist der Verlust unverändert, im überwachten Szenario wird auf den Verlus der BinaryCrossEntropyLoss des Klassifikator aufaddiert<jupyter_code>activeAD = ED_Feedback(vaeAD)
keras.utils.plot_model(activeAD.get_supervised(), show_shapes=True)
#keras.utils.plot_model(vae_al, show_shapes = True)
def attention_visualizer():
    
    (orig, _), (_, _) = aursad()
    
    attention_vis = Model(inputs=encoder_input, outputs=attention_scores)
    
    scores = attention_vis.predict(orig[:batch_size,:,:])[0,:,:]
    scores = np.log(scores)
    print("scores: ",scores.shape)
    #plt.imshow(scores, cmap='hot', interpolation='nearest')
    #plt.show()
    
    n = scores.shape[0]
    n = n//10
    
    print(np.sum(scores, axis=1))
    
    fig, (ax1, ax2) = plt.subplots(1,2,gridspec_kw={'width_ratios': [3, 1]}, figsize=(10,6))
    im = ax1.imshow(scores)
    
    ax1.set_xticks(np.arange(10)*n)
    ax1.set_yticks(np.arange(10)*n)

    # Loop over data dimensions and create text annotations.
    for i in range(10):
        for j in range(10):
            text = ax1.text(j*n, i*n, np.round(scores[i*n, j*n],2), ha="center", va="center", color="w")

    ax1.set_title("Attention heat map")
    
    ax2.plot(orig[0,:,0],-np.arange(0,len(orig[0,:,0])))
    ax2.set_ylim(-len(orig[0,:,0]),0)
    
    fig.tight_layout()
    plt.show()
   
#attention_visualizer()
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import numpy as np

def color_from_label(labels):
    l2c = {0:'tab:cyan',
           1:'tab:blue',
           2:'tab:green',
           3:'tab:olive',
           4:'tab:orange',
           5:'tab:red',
           6:'tab:brown',
           7:'tab:purple',
           8:'tab:pink',
           9:'tab:gray',
          }
    return [l2c[label] for label in labels]


def latent_space_pca(dim=2, dataset='train'):

    x, y = supervised(dataset=dataset)
    pca = PCA(n_components=dim)

    z = encoder.predict(x)
    z_pca = pca.fit_transform(z)

    if dim==3:
        from mpl_toolkits import mplot3d
        fig = plt.figure(figsize=(8,8))
        ax = plt.axes(projection='3d')
        ax.scatter3D(z_pca[:,0], z_pca[:,1], z_pca[:,2], c=color_from_label(y), alpha=0.5)

    plt.figure(figsize=(15,15))
    plt.scatter(z_pca[:,0], z_pca[:,1], c=color_from_label(y))

def visualize_latent_space(show_size=False, dataset='test'):

    x, y = supervised(dataset=dataset)

    z = encoder.predict(x)
    z_1 = z[:,0]
    z_2 = z[:,1]

    plt.figure(figsize=(15,15))
    plt.scatter(z_1, z_2, c=color_from_label(y))

#visualize_latent_space()
latent_space_pca(dim=2)
def mahalanobis_anomaly_score_series(x_true, x_pred):
    
    a = np.zeros((x_true.shape[0], x_true.shape[1]))
    for t in range(x_true.shape[1]):
        
        xt_true = x_true[:,t,:]
        xt_pred = x_pred[:,t,:]
        
        error = np.abs(xt_true-xt_pred)
    
        mu = np.mean(error, axis=0)
        #print(mu.shape)
        
        cov = np.cov(error, rowvar=False)
        inv_cov = np.linalg.inv(cov)
        #print(cov.shape)
        
        for s in range(x_true.shape[0]):
            e = error[s,:]

            a[s,t] = np.matmul(np.matmul((e-mu).T, inv_cov),(e-mu))
        
    return a

def show_anomaly_score(sample=0, label=0):
    
    orig, y = supervised()    
    x_pred = vae.predict(orig)

    a = mahalanobis_anomaly_score_series(orig, x_pred)
    
    s = 0
    while y[s] != label:
        s+=1   
            
    num_plots = orig.shape[2]+1
    fig, ax = plt.subplots(num_plots,1,figsize=(15,num_plots*3))
        
    for i, axi in enumerate(ax):
            
        if axi != ax[-1]:
            axi.plot(orig[s,:,i])
            axi.plot(x_pred[s,:,i])
        else:
            break
        
    ax[-1].plot(a[s,:], c="r")
    ax[-1].hlines(np.mean(a[s,:]), 0,400)
    ax[-1].legend(["mahalanobis", str(np.mean(a[s,:]))])
    plt.show()
        
show_anomaly_score(label=0)
#def load_vae(path):
#    mapping = {
#        "KLDivergenceLayer": KLDivergenceLayer,
#        "SampleLayer": SampleLayer,
#        "ReconstructionLossLayer": ReconstructionLossLayer,
#    }
#    vae = keras.models.load_model(path, custom_objects = mapping)
#    return vae
def mse(x_true, x_pred):
        
        loss = np.mean((x_true-x_pred)**2, axis=1)
        loss = np.mean(loss, axis=-1)
        return loss
    
def mae(x_true, x_pred):
    loss = np.mean(np.abs(x_true-x_pred), axis=1)
    loss = np.mean(loss, axis=-1)
    return loss

def mahalanobis_anomaly_score(x_true, x_pred):
    """Reduces the mahalanobis anomaly score series to a scalar"""
    series = mahalanobis_anomaly_score_series(x_true, x_pred)
    return np.mean(series, axis=1)

def anomaly_detection():
    
    x_test, y_test = supervised(dataset="test")
    x_pred = vae.predict(x_test)

    anomaly_scores = mahalanobis_anomaly_score(x_test, x_pred)

    all_scores = list()
    labels = [i for i in range(4)]

    for label in labels:
        scores = anomaly_scores[y_test== label]
        if len(scores) == 0:
            all_scores.append([0])
        else:
            all_scores.append(scores)
    
    fig, ax = plt.subplots(figsize=(8,12))
    ax.boxplot(all_scores,
        vert=True,  # vertical box alignment
        patch_artist=True,  # fill with color
        labels=labels)
    ax.set_ylabel("anomaly score")
    
    x_train, y = unsupervised()
    print(x_train.shape)
    print(np.sum(y!=0))
    
    x_pred = vae.predict(x_train)
    anomaly_scores = mahalanobis_anomaly_score(x_train, x_pred)

    critical_anomaly_score = np.quantile(anomaly_scores, 0.98)
    print(critical_anomaly_score)
    
    ax.axhline(critical_anomaly_score, color="r")

#my_vae = load_vae(model_str)
anomaly_detection()
def quantile_threshold(p, loss_func=mae, return_loss=False):
    
    x, _ = supervised(dataset='train')
    x_hat = vae.predict(x)

    loss = loss_func(x, x_hat)
    threshold = np.quantile(loss, p)

    if return_loss:
        return threshold, loss
    else:
        return threshold

def confusion_matrix(actual, predicted):
    """Computes the elements of a confusion matrix: tp, fp, fn, tn
    """

    # positiv = anomalous, negative = normal
    tp = np.sum(np.logical_and(actual == 1, predicted == 1))
    tn = np.sum(np.logical_and(actual == 0, predicted == 0))
    fp = np.sum(np.logical_and(actual == 0, predicted == 1))
    fn = np.sum(np.logical_and(actual == 1, predicted == 0))
    
    return tp, fp, fn, tn

def accuracy(tp, fp, fn, tn):  
    return (tp+tn) / (tp+fp+fn+tn)

def precision(tp, fp, fn, tn):
    return tp/(tp+fp)

def recall(tp, fp, fn, tn):
    return tp/(tp+fn)

def f_score(tp, fp, fn, tn):
    p = precision(tp, fp, fn, tn)
    r = recall(tp, fp, fn, tn)
    return 2*p*r/(p+r)

def mcc_score(tp, fp, fn, tn):
    return (tp*tn-fp*fn)/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))

def baseline_ad(model = "unsupervised", loss_func=mae):
    
    ##x_train, _ = unsupervised(percentage=0.125)
    #x_train, _ = supervised(dataset="train")
    #    
    #x_pred = vae.predict(x_train)
    #training_loss = loss_func(x_train, x_pred)    
    #    
    #critical_anomaly_score = np.quantile(training_loss, 0.865)

    critical_anomaly_score = quantile_threshold(0.865, loss_func)
    
    print(f'Critical Anomaly Score = {critical_anomaly_score}')
    
    x_test, y_test = supervised(dataset="test")
    
    for i in range(5):
        count = np.sum(y_test==i)
        print(f'Anzahl {i}: {count}')
    
    x_test_pred = vae.predict(x_test)
    test_loss = loss_func(x_test, x_test_pred)    
    
    predicted = test_loss > critical_anomaly_score
    actual = y_test != 0
    
    tp, fp, fn, tn = confusion_matrix(actual, predicted)
    n = len(y_test)
    
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}, N = {n}')
    
    acc = accuracy(tp, fp, fn, tn)
    prec = precision(tp, fp, fn, tn)
    rec = recall(tp, fp, fn, tn)
    f1 = f_score(tp, fp, fn, tn)
    mcc = mcc_score(tp, fp, fn, tn)
    print(f'Accuracy = {acc}, F1 = {f1}, precision = {prec}, recall = {rec}, mcc = {mcc}')
    
    
baseline_ad(loss_func=mahalanobis_anomaly_score)<jupyter_output><empty_output><jupyter_text>Query Strategy<jupyter_code>def naive_query(budget, return_idx=False):
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0

    x = x[:budget,:,:]  
    y = y[:budget]   
    
    if return_idx:
        return (x,y), np.arange(budget)
    else:
        return x, y
    
def edgecase_query(budget, return_idx=False):
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0

    threshold, score = quantile_threshold(0.865,
                            loss_func=mahalanobis_anomaly_score, return_loss=True)
    #print(f'Threshold anomaly score: {threshold}')
        
    edge_cases = np.abs(score - threshold)    
    idx = np.argsort(edge_cases)
    idx = idx[:budget]
        
    if return_idx:
        return (x[idx,::], y[idx]), idx
    else:
        return x[idx,::], y[idx]
    
def most_anomalous_query(budget, return_idx=False):
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0
    
    x_hat = vae.predict(x)
    score = mahalanobis_anomaly_score(x, x_hat)

    idx = np.argsort(score)
    idx = idx[-budget:]
        
    if return_idx:
        return (x[idx,::], y[idx]), idx
    else:
        return x[idx,::], y[idx]

def normal_query(budget, return_idx=False):
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0
    
    x_hat = vae.predict(x)
    score = mahalanobis_anomaly_score(x, x_hat)

    idx = np.argsort(score)
    idx = idx[:budget]
        
    if return_idx:
        return (x[idx,::], y[idx]), idx
    else:
        return x[idx,::], y[idx]


def clustering_query(budget, plot=False, return_idx=False):
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0
      
    def dbscan_clustering(z, sample_weight=None):
        y = DBSCAN(min_samples=9, eps=0.1).fit_predict(z, sample_weight=sample_weight)
        return y
    
    def kmeans_clustering(z, sample_weight=None):
        kmeans = KMeans(n_clusters=10, random_state=0).fit(z, sample_weight=sample_weight)
        return kmeans.predict(z)
    

    threshold, score = quantile_threshold(0.865,
                            loss_func=mahalanobis_anomaly_score, return_loss=True)

    sample_weight = score > threshold
    sample_weight = sample_weight * 3 + 1

    pca = PCA(n_components=2)
    z = encoder.predict(x)
    z_pca = pca.fit_transform(z)

    y_cluster = dbscan_clustering(z_pca, sample_weight=sample_weight)
    
    import pandas as pd
    y_series = pd.Series(y_cluster)

    #print(y_series)
    grouped = y_series.groupby(y_series).count().reset_index()
    #print(grouped.columns)
    grouped=grouped.sort_values(0)
    #print(grouped)

    counts = grouped[0]
    #print(counts)

    idx = list()
    for j, (i, _) in enumerate(counts.iteritems()):
        
        remaining = budget - len(idx)
        samples_per_label = int(np.ceil(remaining/(len(counts)-j)))
        
        pos = np.arange(len(y_series))
        new_idx = pos[y_series==(i-1)][:samples_per_label]
        idx.extend(new_idx)
        #print(i,_,len(new_idx))

    if plot:
        print(np.unique(y_cluster))
        fig = plt.figure(figsize=(12,12))
        plt.scatter(z_pca[:,0], z_pca[:,1],c=y_cluster, s=100, cmap=cm.tab20)
        z_neg = z_pca[y_cluster<0,:]
        plt.scatter(z_neg[:,0], z_neg[:,1],c='k', s=100)
        plt.scatter(z_pca[idx,0], z_pca[idx,1], edgecolors='r', s=160, marker='o', facecolors='none')

    if return_idx:
        return (x[idx,::], y[idx]), idx
    else:
        return x[idx,::], y[idx]
    
#_,_ = clustering_query(128, plot=True)<jupyter_output><empty_output><jupyter_text>E2: Vergleich der AbfragestrategienFür die Ergebnisse des CNN-LSTM Modell werden die verschiedenen Abfragestrategien verglichenDarstellungen:* a - Quantitativ: Wie viele Beispiele der einzelnen Klassen werden ausgewählt?* b - Qualitativ: Welche Beispiele (Lage im latenten Raum) der einzelnen Klassen werden ausgewählt?* c - Wirkung: Welche Leistungsfähigkeit des überwachten Modells ist damit erreichbar?<jupyter_code>def e2a():
    budget = 128

    queries = [naive_query, edgecase_query, most_anomalous_query, normal_query, clustering_query]
    for query in queries:
        qx, qy = query(budget)

        num_normal = np.sum(qy == 0)
        num_anomalies = np.sum(qy != 0)
        print(f"Query: {query.__name__}, Normal: {num_normal}, anomalies {num_anomalies}")

e2a()
from matplotlib.gridspec import GridSpec
from matplotlib.colors import Normalize
import pandas as pd

def e2b():
    # query the entire, non-restricted 
    x, y = unsupervised(percentage=0.99)
    y = y != 0
    
    pca = PCA(n_components=2)
    z = encoder.predict(x)
    z_pca = pca.fit_transform(z)

    x_hat = vae.predict(x)
    threshold, score = quantile_threshold(0.865,
                            loss_func=mahalanobis_anomaly_score, return_loss=True)
    
    # make the scores logarithmic for a enhanced depiction
    threshold, score = np.log(threshold), np.log(score)

    (_,_), idx = most_anomalous_query(budget=64, return_idx=True)

    fig= plt.figure(figsize=(12,8))
    grid_spec = fig.add_gridspec(ncols=2, nrows=2, height_ratios=[8,1], width_ratios=[9.8, 0.2])

    ax1 = fig.add_subplot(grid_spec[0,0])
    ax2 = fig.add_subplot(grid_spec[1,:])
    axcb = fig.add_subplot(grid_spec[0,1])

    ax1.scatter(z_pca[:,0], z_pca[:,1], c=score, cmap=cm.RdYlGn_r)
    ax1.scatter(z_pca[idx,0], z_pca[idx,1], edgecolors='r', marker='o', facecolors='none', s=160)

    clb = fig.colorbar(cm.ScalarMappable(cmap=cm.RdYlGn_r, norm=Normalize(vmin=np.min(score), vmax=np.max(score))), cax=axcb)
    clb.set_label('log(Anomaliescore)', fontsize=14)
    ax2.scatter(score, np.ones(len(score)), s=20, c='k', label='Beispiel')
    ax2.set_yticks([])
    ax2.scatter(score[idx], np.ones(len(idx)), edgecolors='r', marker='o', facecolors='none', s=100, label='abgefragt')
    ax2.vlines(threshold, 0.95, 1.05, linestyle="--", color="tab:orange", label="Schwellenwert")
    ax2.set_xlabel("log(Anomaliescore)", fontsize=14)
    ax2.legend(ncol=3, loc='upper center', bbox_to_anchor=(0.5,-0.6), fontsize=14)
    plt.tight_layout()

    #df1 = pd.DataFrame({'score':np.log(score), 'queried':["Beispiele"]*len(score)})
    #df2 = pd.DataFrame({'score':np.log(score[idx]), 'queried':["ausgewählt"]*len(score[idx])})
    #df = pd.concat([df1, df2])
    #df['all'] = ''
    #ax = sns.violinplot(y='all', x='score', data=df, hue='queried', 
    #        split=True, palette="Set2", scale='count', inner=None)

# e2b()
def active_learning(x_query, y_query, strategy='none'):
    
    budget = len(y_query)
    model_file = os.path.join(output_path, f'b{budget}-s{strategy}-{model_str}')
    y_query = keras.utils.to_categorical(y_query, num_classes=2)

    callbacks = [
        #keras.callbacks.ModelCheckpoint(
        #    monitor = 'loss',
        #    filepath = model_file, 
        #    save_weights_only = False,
        #    save_best_only = True),
        keras.callbacks.EarlyStopping(monitor='loss', patience=8)
        ]

    # warm-up
    #vae.fit(x_query, x_query, epochs=1, shuffle=True, batch_size=batch_size)

    history = activeAD.get_supervised().fit(x_query, y_query, shuffle = True, 
                epochs=180, batch_size=64, callbacks=callbacks)
    #activeAD.get_supervised().load_weights(model_file)
    
    plot_history(history, ['loss'])
    with open(os.path.join(output_path, "active-history-1.json"), 'w') as file:
            json.dump(history.history, file)

    x_test, y_test = supervised(dataset = "test")
    y_test = y_test != 0
    y_test = keras.utils.to_categorical(y_test, num_classes=2)
    
    predicted = activeAD.get_supervised().predict(x_test)
    predicted = np.argmax(predicted, axis=-1)
    actual = np.argmax(y_test, axis=-1)

    tp, fp, fn, tn = confusion_matrix(actual, predicted)

    tp, fp, fn, tn = confusion_matrix(actual, predicted)
    acc = accuracy(tp, fp, fn, tn)
    prec = precision(tp, fp, fn, tn)
    rec = recall(tp, fp, fn, tn)
    f1 = f_score(tp, fp, fn, tn)
    mcc = mcc_score(tp, fp, fn, tn)
        
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}')
    print(f'Accuracy = {acc}, F1 = {f1}, precision = {prec}, recall = {rec}, mcc = {mcc}')

def e2c():
    qx1,qy1 = normal_query(budget=128)
    qx2,qy2 = most_anomalous_query(budget=256)

    qx = np.concatenate((qx1, qx2), axis=0)
    qy = np.concatenate((qy1, qy2), axis=0)
    
    active_learning(qx, qy, strategy='M')

e2c()<jupyter_output><empty_output><jupyter_text>E3: Feedback zur F-Score OptimieruneVorteile F-Score:* einfach zu berechnen* gebräuchig für ML* Wertebereich [0,1]* Goodfellow, p. 423f* Hand, DavidVorteile MCC:* unabhängig, welche Klasse als positiv / negativ deklariert wird* basiert direkt auf der Konfusionsmatirx, unabhängig von Imbalance<jupyter_code>def e3():
    xq, yq = naive_query(budget=256) 
    xq_hat = vae.predict(xq)
    score = mahalanobis_anomaly_score(xq, xq_hat)

    minScore = np.min(score)
    maxScore = np.max(score)
    scores = np.arange(minScore, maxScore, 0.01)

    opt = -1
    threshold = None
    for s in scores:
        y_pred = score > s
        tp, fp, fn, tn = confusion_matrix(yq, y_pred)
        new = f_score(tp, fp, fn, tn)
        if new > opt:
            opt = new
            threshold=s
    
    # report
    print(f'opt. score (query dataset): {opt} @ opt. threshold: {threshold}')

    # final evaluation on test data
    x_test, y_test = supervised(dataset='test')
    actual = y_test != 0

    x_test_hat = vae.predict(x_test)
    test_score = mahalanobis_anomaly_score(x_test, x_test_hat)

    predicted = test_score > threshold

    tp, fp, fn, tn = confusion_matrix(actual, predicted)
    acc = accuracy(tp, fp, fn, tn)
    prec = precision(tp, fp, fn, tn)
    rec = recall(tp, fp, fn, tn)
    f1 = f_score(tp, fp, fn, tn)
    mcc = mcc_score(tp, fp, fn, tn)
        
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}')
    print(f'Accuracy = {acc}, F1 = {f1}, precision = {prec}, recall = {rec}, mcc = {mcc}')

#e3()<jupyter_output><empty_output><jupyter_text>E4: Vergleich der ArchitekturenFür die Abfragestrategie `most_anomalous_query` ergänzt um die `normal_query` werden die verschiedenen Architekturen bei B=256 verglichen.<jupyter_code>def e4():
    qx1,qy1 = normal_query(budget=128)
    qx2,qy2 = most_anomalous_query(budget=256)

    qx = np.concatenate((qx1, qx2), axis=0)
    qy = np.concatenate((qy1, qy2), axis=0)
    
    active_learning(qx, qy, strategy='MN')
#e4()<jupyter_output><empty_output><jupyter_text>Restrukturierung des latenten Raums<jupyter_code>def z_shift():
    x, y = supervised(dataset='train')
    z_before = encoder.predict(x)

    qx1,qy1 = normal_query(budget=128)
    qx2,qy2 = most_anomalous_query(budget=256)

    qx = np.concatenate((qx1, qx2), axis=0)
    qy = np.concatenate((qy1, qy2), axis=0)
    
    active_learning(qx, qy, strategy='MN')

    z_after = encoder.predict(x)

    return z_before, z_after, y

z_b, z_a, groundtruth = z_shift()
def plot_z_shift(z_b, z_a, y):
    fig, axs = plt.subplots(ncols=2, figsize=(12,6))

    def single_view(ax, z, y):

        pca = PCA(n_components=2)
        z_pca = pca.fit_transform(z)

        ax.scatter(z_pca[:,0], z_pca[:,1], c=y)

    single_view(axs[0], z_b, y)
    single_view(axs[1], z_a, y)

plot_z_shift(z_b, z_a, groundtruth)

assert False
qx1,qy1 = normal_query(budget=128)
qx2,qy2 = most_anomalous_query(budget=256)

qx = np.concatenate((qx1, qx2), axis=0)
qy = np.concatenate((qy1, qy2), axis=0)

print(np.sum(qy1!=0), np.sum(qy2!=0))
print(np.shape(qx))
#vae.fit(qx, qx, epochs=1)
def active_learning(x_query, y_query):
    
    y_query = keras.utils.to_categorical(y_query, num_classes=2)
    
    
    #callbacks = [keras.callbacks.ModelCheckpoint(
    #    monitor = 'loss',
    #    filepath = 'vae_al_ef0-{epoch:02d}.hdf5', 
    #    save_weights_only = False,
    #    save_best_only = False)]

    history = activeAD.get_supervised().fit(x_query, y_query, shuffle = True, epochs=40, batch_size=32)
    
    x_test, y_test = supervised(dataset = "test")
    y_test = y_test != 0
    y_test = keras.utils.to_categorical(y_test, num_classes=2)
    
    predicted = activeAD.get_supervised().predict(x_test)
    #print(predicted)

    predicted = np.argmax(predicted, axis=-1)
    actual = np.argmax(y_test, axis=-1)

    #print(predicted.shape, predicted)
    #print(actual.shape, actual)
    
    # positiv = anomalous, negative = normal
    n = len(y_test)
    tp = np.sum(np.logical_and(actual == 1, predicted == 1))
    tn = np.sum(np.logical_and(actual == 0, predicted == 0))
    fp = np.sum(np.logical_and(actual == 0, predicted == 1))
    fn = np.sum(np.logical_and(actual == 1, predicted == 0))
    
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}, N = {n}')
      
    accuracy = (tp+tn) / n
    precision = tp / (tp+fp)
    recall = tp / (tp+fn)
    f1 = 2 * precision * recall / (precision + recall)
    print(f'Accuracy = {accuracy}, F1 = {f1}, precision = {precision}, recall = {recall}')

#vae.load_weights(model_str)  
active_learning(qx, qy)
def active_learning_modelfeedback(normal = 0):
    budget = batch_size 
    
    x_query, y_query = most_anomalous_query(budget)
    print(np.sum(y_query))
    
    callbacks = [keras.callbacks.ModelCheckpoint(
        monitor = 'loss',
        filepath = 'vae_al_ef0-{epoch:02d}.hdf5', 
        save_weights_only = False,
        save_best_only = False)]

    history = vae_al.fit(x_query, y_query, shuffle = True, epochs=60, batch_size=32)
    
    x_test, y_test = supervised(dataset = "test")
    
    predicted = vae_al.predict(x_test).reshape(-1) > 0.5
    actual = (y_test != normal)
    
    # positiv = anomalous, negative = normal
    tp = np.sum(np.logical_and(actual == 1, predicted == 1))
    tn = np.sum(np.logical_and(actual == 0, predicted == 0))
    fp = np.sum(np.logical_and(actual == 0, predicted == 1))
    fn = np.sum(np.logical_and(actual == 1, predicted == 0))
    n = len(actual)
    
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}, N = {n}')
    
    accuracy = (tp+tn) / n
    precision = tp / (tp+fp)
    recall = tp / (tp+fn)
    f1 = 2 * precision * recall / (precision + recall)
    print(f'Accuracy = {accuracy}, F1 = {f1}, precision = {precision}, recall = {recall}')
    
#active_learning_modelfeedback()
def active_learning_decoupled(normal = 0):
    """Enhance the prediction accuracy by retraining the model with the top-k most intersting samples in a
    supervised fashion.
        1) Find the top-k samples and get their labels
        2) Retrain the model with these samples in a supervised fashion -> prevent overfitting
        3) Evaluate accuracy, F1-score, ..."""
    
    budget = batch_size  
    
    x_query, y_query = most_anomalous_query(budget)
    print(np.sum(y_query))
    
    callbacks = [keras.callbacks.ModelCheckpoint(
        monitor = 'loss',
        filepath = 'vae_al0-{epoch:02d}.hdf5', 
        save_weights_only = False,
        save_best_only = False)]
    
    z_query = encoder.predict(x_query)

    history = vae_al.fit(z_query, y_query, shuffle = True, epochs=100, batch_size=32)
        #training_data,
        #shuffle=True,      
     #   callbacks = callbacks,
      #  validation_data = (x_valid, y_valid))
    
    x_test, y_test = supervised(dataset = "test")
    z_test = encoder.predict(x_test)
    
    predicted = vae_al.predict(z_test).reshape((-1)) > 0.5
    actual = y_test != normal
    
    # positiv = anomalous, negative = normal
    tp = np.sum(np.logical_and(actual == 1, predicted == 1))
    tn = np.sum(np.logical_and(actual == 0, predicted == 0))
    fp = np.sum(np.logical_and(actual == 0, predicted == 1))
    fn = np.sum(np.logical_and(actual == 1, predicted == 0))
    n = len(y_test)
    
    print(f'TP = {tp}, TN = {tn}, FP = {fp}, FN = {fn}, N = {n}')
    
    accuracy = (tp+tn) / n
    precision = tp / (tp+fp)
    recall = tp / (tp+fn)
    f1 = 2 * precision * recall / (precision + recall)
    print(f'Accuracy = {accuracy}, F1 = {f1}, precision = {precision}, recall = {recall}')
    
#active_learning_decoupled()
visualize_latent_space(None, show_size = True)
anomaly_detection(vae)
a, b = supervised(dataset="train")
len(b)
np.sum(b!=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1467088967393511, 'fraction_numerical': 0.019937467007755717, 'mean_word_length': 3.110832916040728, 'pattern_counts': {'":': 3, '<': 26, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '194938', 'license': 'MIT', 'n_tokens_mistral': 9916, 'n_tokens_neox': 9460, 'n_words': 1816, 'path': 'DAAD-AURSAD-opt.ipynb', 'repo_name': 'dmholtz/deep-anomaly-detection', 'size': '24627'}