<jupyter_start><jupyter_text>Optimierung und Evaluierung des Sequential bow Modells<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sn
import h5py
import pydot

from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report, log_loss
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier

from wordcloud import WordCloud
from PIL import Image
from PIL import ImageFilter

from hyperopt import Trials, STATUS_OK, tpe
from hyperas import optim
from hyperas.distributions import choice, uniform, normal, qlognormal, randint

from keras.models import Sequential
from keras.layers import Dense, Dropout, LeakyReLU, Activation
from keras.regularizers import l2, l1
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping
from keras.models import load_model
from keras.utils import np_utils, plot_model
from keras.optimizers import Adagrad
from keras.losses import binary_crossentropy
def data_bow():
    x_test = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="test")
    x_train = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="train")
    y_test = x_test.label
    y_train = x_train.label
    x_test= x_test.drop('label',axis=1)
    x_train = x_train.drop('label',axis=1)
    return x_train, y_train, x_test, y_test<jupyter_output><empty_output><jupyter_text>First Test<jupyter_code>x_test = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="test")
x_train = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="train")
y_test = x_test.label
y_train = x_train.label
x_test= x_test.drop('label',axis=1)
x_train = x_train.drop('label',axis=1)
dim = x_train.shape[1]
model = Sequential()
model.add(Dense(1770, kernel_regularizer=l1(2.0036577552673407e-06), input_dim=dim))
model.add(Activation('relu'))
model.add(Dense(9,kernel_regularizer=l2(0.05407632514834404)))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy', metrics=['accuracy'],

            optimizer='Adagrad')

model.summary()
    
result = model.fit(x_train.values, y_train.values,
                    batch_size=64,
                    epochs=100,
                    verbose=2,
                    validation_split=0.3)
def plot_history(network_history):
    plt.figure()
    plt.xlabel('Epochen')
    plt.ylabel('Verlust')
    plt.plot(network_history.history['loss'])
    plt.plot(network_history.history['val_loss'])
    plt.legend(['Training', 'Validierung'])
    plt.show()
    plt.close()
plot_history(result)<jupyter_output><empty_output><jupyter_text>Analyse des Inputs<jupyter_code>x_test = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="test")
x_train = pd.read_hdf("../build/preprocessed/bow_data_500.hdf5",key="train")
X = x_test.append(x_train)<jupyter_output><empty_output><jupyter_text>$$max\left(\frac{(\bar{w_1}-\bar{w_2})^2}{s_1^2+s_2^2}\right)$$<jupyter_code>mu_real = X[X.label==1].mean()
mu_fake = X[X.label==0].mean()
sorted_words = ((mu_real-mu_fake)**2/(np.var(X[X.label==1])**2 + np.var(X[X.label==0])**2)).sort_values(ascending=False).index

words_plot = pd.DataFrame({'Fake':X[X.label==0][sorted_words[1:11]].mean(),'Real':X[X.label==1][sorted_words[1:11]].mean()})
words_plot.plot(kind='bar')
plt.ylabel(r"$\overline{w}$")
plt.legend()
plt.tight_layout()
plt.savefig("../build/plots/data_visualisation.pdf")
plt.show()
news = pd.read_csv('../data/mixed_news/news_dataset.csv')
news = news.dropna(subset=['title','content'])
news = news[news.content != ' ']
news = news[news.title != ' ']
text_len_real  =  [len(c) for c in news[news['label']=='real'].content]
text_len_fake = [len(c) for c in news[news['label']=='fake'].content]
print("Mittlere textlänge Real: ",np.mean(text_len_real))
print("Mittlere Textlänge Fake: ",np.mean(text_len_fake))
print("Real ist %d länger wie Fake: " % (np.mean(text_len_real)-np.mean(text_len_fake)))<jupyter_output><empty_output><jupyter_text>How could the Hyperparameter be distributed StruktureGröße der ersten und zweiten hidden layer<jupyter_code>x = np.round(np.random.lognormal(6,0.5,10000)/10)*10
plt.hist(x,bins=100)
plt.xlim(0,5000)
plt.show()<jupyter_output><empty_output><jupyter_text>Größer der dritten hidden layer<jupyter_code>x = np.round(np.random.lognormal(4,0.5,10000)/1)*1
plt.hist(x,bins=100)
plt.show()<jupyter_output><empty_output><jupyter_text>Regularization<jupyter_code>x =np.random.uniform(0,0.1,10000)
plt.hist(x,bins=100)
plt.show()<jupyter_output><empty_output><jupyter_text>Model creation<jupyter_code>def model_structure(x_train, y_train, x_test, y_test):
    dim = x_train.shape[1]
    model = Sequential()
    model.add(Dense(int({{qlognormal(6,0.5,10)}}), input_dim=dim))
    model.add(Activation('relu'))
    
    if {{choice(['three', 'four'])}} == 'four':
        model.add(Dense(int({{qlognormal(6,0.5,10)}})))
        model.add(Activation('relu'))
        
    model.add(Dense(int({{qlognormal(4,0.5,1)}})))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy', metrics=['accuracy'],
                  optimizer='adam')

    result = model.fit(x_train.values, y_train.values,
              batch_size=64,
              epochs=30,
              verbose=2,
              validation_split=0.3)
    validation_acc = np.amax(result.history['val_acc']) 
    print('Best validation acc of epoch:', validation_acc)
    return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def model_training(x_train, y_train, x_test, y_test):
    dim = x_train.shape[1]
    model = Sequential()
    model.add(Dense(1770, input_dim=dim))
    model.add(Activation('relu'))
    model.add(Dense(9))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss={{choice(['hinge','binary_crossentropy','squared_hinge'])}}, metrics=['accuracy'],
                  optimizer={{choice(['adam','AdaDelta','Adagrad'])}})

    result = model.fit(x_train.values, y_train.values,
              batch_size=64,
              epochs=30,
              verbose=2,
              validation_split=0.3)
    validation_acc = np.amax(result.history['val_acc']) 
    print('Best validation acc of epoch:', validation_acc)
    return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def model_regularization(x_train, y_train, x_test, y_test):
    dim = x_train.shape[1]
    model = Sequential()
    model.add(Dense(1770, kernel_regularizer=l1({{uniform(0,0.1)}}), input_dim=dim))
    model.add(Activation('relu'))
    model.add(Dense(9,kernel_regularizer=l2({{uniform(0,0.1)}})))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy', metrics=['accuracy'],
                  optimizer='Adagrad')

    result = model.fit(x_train.values, y_train.values,
                      batch_size=64,
                      epochs=30,
                      verbose=2,
                      validation_split=0.3)
    validation_acc = np.amax(result.history['val_acc']) 
    print('Best validation acc of epoch:', validation_acc)
    return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def model_optimizer(x_train, y_train, x_test, y_test):
    dim = x_train.shape[1]
    model = Sequential()
    model.add(Dense(1770, kernel_regularizer=l1(2.0*10**(-6)), input_dim=dim))
    model.add(Activation('relu'))
    model.add(Dense(9,kernel_regularizer=l2(0.05407632514834404)))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy', metrics=['accuracy'],
                  optimizer=Adagrad(lr={{uniform(0,1)}}, epsilon=None, decay={{uniform(0,1)}}))

    result = model.fit(x_train.values, y_train.values,
                      batch_size=64,
                      epochs=30,
                      verbose=2,
                      validation_split=0.3)
    validation_acc = np.amax(result.history['val_acc']) 
    print('Best validation acc of epoch:', validation_acc)
    return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}<jupyter_output><empty_output><jupyter_text>Optimization with hyperoptAlgorithm: Tree of Parzen EstimatorsOptimierung in 3 Schritten:    - Struktur (Tiefe (2 oder 3 hidden Layers) und Breite)    - Training (loss function und optimizer)    - Regularizierung ( L1 für die erste Layer und L2 für 2 und 3)<jupyter_code>trials = Trials()
best_run, best_model = optim.minimize(model=model_structure,
                                      data=data_bow,
                                      algo=tpe.suggest,
                                      max_evals=50,
                                      trials=trials,
                                     notebook_name='Sequential_bow')
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save('../model/best_Hyperopt_NN_bow_struct_500.hdf5')
trials = Trials()
best_run, best_model = optim.minimize(model=model_training,
                                      data=data_bow,
                                      algo=tpe.suggest,
                                      max_evals=15,
                                      trials=trials,
                                     notebook_name='Sequential_bow')
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save('../model/best_Hyperopt_NN_bow_training_500.hdf5')
trials = Trials()
best_run, best_model = optim.minimize(model=model_regularization,
                                      data=data_bow,
                                      algo=tpe.suggest,
                                      max_evals=80,
                                      trials=trials,
                                     notebook_name='Sequential_bow')
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save('../model/best_Hyperopt_NN_bow_regularization2_500.hdf5')
trials = Trials()
best_run, best_model = optim.minimize(model=model_optimizer,
                                      data=data_bow,
                                      algo=tpe.suggest,
                                      max_evals=100,
                                      trials=trials,
                                     notebook_name='Sequential_bow')
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save('../model/best_Hyperopt_NN_bow_optimizer_500.hdf5')<jupyter_output><empty_output><jupyter_text>Beste Regularization: L1 in der ersten Layer = 0.00010911483516010123L2 in der zweiten Layer = 0.027248712155710758lr = 0.12056175158012145epsilon = 0.1999709211230266Die Optimierung des Lernrate führt jedoch zu einem Optimizer, der leicht in Nebenmaxima stecken bleibt (hier das Nebenmaxima, dass alles auf real zu schätzen). Daher wird der Standard Lerner verwendet  Evaluation of best model  Train best modelNeues Training des besten Modells, welches Optimiert bezüglich der Hyperparameter ist<jupyter_code>def plot_history(network_history):
    plt.figure()
    plt.xlabel('Epochen')
    plt.ylabel('Verlust')
    plt.plot(network_history.history['loss'])
    plt.plot(network_history.history['val_loss'])
    plt.legend(['Training', 'Validierung'])
    #plt.show()
    plt.savefig("../build/plots/bow/500/history_bow_best.pdf")
    plt.close()
X_train, Y_train, X_test, Y_test = data_bow()
best_model = load_model('../model/best_Hyperopt_NN_bow_regularization2_500.hdf5')
model = Sequential.from_config(best_model.get_config())
print(model.summary())
model.get_config()
filepath = '../model/best_Hyperopt_NN_bow_trained_500.hdf5'
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True)
#stopping = EarlyStopping(monitor='val_loss', min_delta=0, restore_best_weights=True)
model.compile(loss='binary_crossentropy',
              optimizer='Adagrad',
              metrics=['accuracy'])
history = model.fit(X_train.values, Y_train.values, validation_split=0.3,
                    epochs=100,batch_size=64, callbacks=[checkpoint])#, stopping])
plot_history(history)<jupyter_output><empty_output><jupyter_text>Evaluation of best modelBetrachten des trainierten Modells. Darstellung der Confusion Matrix, Overtraining Plot und ROC Curve<jupyter_code>best_model = load_model('../model/best_Hyperopt_NN_bow_trained_500.hdf5')
y_pred = best_model.predict(X_test.values, batch_size=64, verbose=1)
y_pred_train = best_model.predict(X_train.values, batch_size=64, verbose=1)
y_pred_bool = np.round(y_pred[:,0])
Y_test = pd.DataFrame({"label":Y_test,"prediction":y_pred[:,0],"prediction_bool":y_pred_bool})
Y_train = pd.DataFrame({"label":Y_train,"prediction":y_pred_train[:,0]})
print(classification_report(Y_test['label'], Y_test['prediction_bool']))
print("Binary Cross Entropie: ",log_loss(Y_test.label, Y_test.prediction))

#Confusion Matrix
cnfn_matrix = pd.crosstab(Y_test['label'], Y_test['prediction_bool'], rownames=['Wahrheit'], colnames=['Schätzung'])
print(cnfn_matrix)
cnfn_matrix.columns = ['Fake','Real']
cnfn_matrix = cnfn_matrix.rename_axis("Schätzung", axis="columns")
cnfn_matrix.rename(index = {0.0: "Fake", 1.0:'Real'}, inplace = True) 
cnfn_matrix = cnfn_matrix/Y_test.shape[0]
sn.heatmap(cnfn_matrix, annot=True , cmap='viridis')
#plt.show()
plt.savefig("../build/plots/bow/500/cnfsn_mtx_bow_best_nn.pdf")
plt.close()

#Overtraining test
plt.hist(Y_test.prediction[Y_test.label == 0],label="fake test", alpha = 0.4, color = "r",density=True)
plt.hist(Y_train.prediction[Y_train.label == 0],label='fake train', alpha = 0.4, color = 'r', histtype='step',density=True)
plt.hist(Y_test.prediction[Y_test.label == 1],label = "real test",alpha = 0.4, color = "b",density=True)
plt.hist(Y_train.prediction[Y_train.label == 1],label='real train', alpha = 0.4, color = 'b', histtype='step',density=True)

plt.xlabel("Geschätzte Likelihood")
plt.ylabel("Dichte")
plt.legend(loc='upper center')
#plt.show()
plt.savefig("../build/plots/bow/500/prob_bow_best_nn.pdf")
plt.close()

fpr = dict()
tpr = dict()
roc_auc = dict()
fpr, tpr, _ = roc_curve(Y_test.label, Y_test.prediction)
roc_auc = auc(fpr, tpr)

plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
         lw=lw, label='ROC Kurve (AUC = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")
#plt.show()
plt.savefig("../build/plots/bow/500/roc_Hyperopt_bow_best_nn.pdf")
plt.close()<jupyter_output>precision    recall  f1-score   support

           0       0.86      0.87      0.87      3658
           1       0.90      0.89      0.89      4713

   micro avg       0.88      0.88      0.88      8371
   macro avg       0.88      0.88      0.88      8371
weighted avg       0.88      0.88      0.88      8371

Binary Cross Entropie:  0.316260521772017
Schätzung   0.0   1.0
Wahrheit             
0          3184   474
1           515  4198<jupyter_text>Interpretation Wordcloud confusion matrixDarstellung der Wordhäufigkeiten in WordClouds für FP,FN,TP,TN getrennt<jupyter_code>FP = Y_test[(Y_test.prediction_bool== 1) & (Y_test.label == 0)]
FN = Y_test[(Y_test.prediction_bool== 0) & (Y_test.label == 1)]
TP = Y_test[(Y_test.prediction_bool== 1) & (Y_test.label == 1)]
TN = Y_test[(Y_test.prediction_bool== 0) & (Y_test.label == 0)]
X_FP = X_test.loc[FP.index]
X_FN = X_test.loc[FN.index]
X_TP = X_test.loc[TP.index]
X_TN = X_test.loc[TN.index]
def plotWordcloud_cnfn(TN,FN,FP,TP):    
    TN = TN.sum().to_dict()
    FN = FN.sum().to_dict()
    FP = FP.sum().to_dict()
    TP = TP.sum().to_dict()
    
    pad = 5
    
    fig = plt.figure(figsize=(15,10),dpi=100)

    ax = plt.subplot(2, 2, 1)
    wordcloud = WordCloud(background_color='black',
                          width=1920,
                          height=1080,
                          mask=np.array(Image.open('../data/pictures/trump_silhouette.png'))
                          ).generate_from_frequencies(TN)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.margins(x=0, y=0)

    ax = plt.subplot(2, 2, 2)
    wordcloud = WordCloud(background_color='black',
                          width=1920,
                          height=1080,
                          mask= np.array(Image.open('../data/pictures/trump_silhouette.png'))
                          ).generate_from_frequencies(FP)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.margins(x=0, y=0)

    ax = plt.subplot(2, 2, 3)
    wordcloud = WordCloud(background_color='black',
                          width=1920,
                          height=1080,
                          mask=np.array(Image.open('../data/pictures/USA.jpg'))
                          ).generate_from_frequencies(FN)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.margins(x=0, y=0)

    plt.subplot(2, 2, 4)
    wordcloud = WordCloud(background_color='black',
                          width=1920,
                          height=1080,
                          mask=np.array(Image.open('../data/pictures/USA.jpg'))
                          ).generate_from_frequencies(TP)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    
    plt.figtext(0.5, 1.09, r"Prediction", {'fontsize': 30},
         horizontalalignment='center',
         verticalalignment='top')
    plt.figtext(0.25, 1.02, r"fake", {'fontsize': 20},
         horizontalalignment='center',
         verticalalignment='bottom',)
    plt.figtext(0.75, 1.02, r"real", {'fontsize': 20},
         horizontalalignment='center',
         verticalalignment='bottom',)
    
    plt.figtext(-0.07, 0.5, r"Actual", {'fontsize': 30},
         horizontalalignment='left',
         verticalalignment='center',
         rotation=90)
    plt.figtext(0.00, 0.75, r"fake", {'fontsize': 20},
         horizontalalignment='right',
         verticalalignment='center',)
    plt.figtext(0.00, 0.25, r"real", {'fontsize': 20},
         horizontalalignment='right',
         verticalalignment='center',)
    
    plt.margins(x=0, y=0)
    plt.tight_layout()
    #plt.show()
    plt.savefig("../build/plots/bow/500/cnfn_wordcloud.pdf", bbox_inches = 'tight')
    plt.close()
plotWordcloud_cnfn(X_TN,X_FN,X_FP,X_TP)<jupyter_output><empty_output><jupyter_text>Wordcloud fake real news<jupyter_code>def plotWordcloud(content,t):
    if(t!=""):
       mask = np.array(Image.open('../data/pictures/'+t))
    else:
        mask=None
        

    content = content.sum().to_dict()
    wordcloud = WordCloud(background_color='black',
                      width=1920,
                      height=1080,
                      mask=mask
                      ).generate_from_frequencies(content)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.margins(x=0, y=0)
X = X_test.append(X_train)
y = Y_test.append(Y_train)
plt.figure(dpi=200)
plotWordcloud(X[y.label==0],"trump_silhouette.png")
#plt.show()
plt.savefig("../build/plots/fake_wordcloud.pdf",bbox_inches='tight',pad_inches = 0)
plt.close()
plt.figure(dpi=200)
plotWordcloud(X[y.label==1],"USA.jpg")
#plt.show()
plt.savefig("../build/plots/real_wordcloud.pdf",bbox_inches='tight',pad_inches = 0)
plt.close()<jupyter_output><empty_output><jupyter_text>Untersuchung der first layerSummieren der Beträge aller Gewichte eines Neurons ohne Offset und anschließende Darstellung in WordCloud<jupyter_code>words = X_test.columns
first_weights = best_model.layers[0].get_weights()[0]
first_weights = pd.DataFrame(first_weights.transpose())
first_weights.columns = words
first_weightabs = np.abs(first_weights)
first_weightsum = first_weightabs.sum(axis=0)
content = np.abs(first_weightsum).to_dict()
wordcloud = WordCloud(background_color='black',
                      width=1920,
                      height=1080
                      ).generate_from_frequencies(content)
plt.figure(dpi=100)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.margins(x=0, y=0)
#plt.show()
plt.savefig("../build/plots/bow/500/weights_wordcloud.pdf")
plt.close()<jupyter_output><empty_output><jupyter_text>Untersuchung der Confusion Matrix mithilfe der wichtigsten 10 Wörter<jupyter_code>sorted_weights = first_weightsum.sort_values(ascending=False)
best_words = sorted_weights[:10].index

fig = plt.figure(figsize=(15,10),dpi=100)

ax = plt.subplot(2, 2, 1)
(X_TN[best_words].sum()/X_TN.shape[0]).plot(kind='bar',label="TN",color='r')
plt.margins(x=0, y=0)
plt.ylabel("mittlere Worthäufigkeit")
plt.ylim(0.01,60)
plt.yscale("log")

ax = plt.subplot(2, 2, 2)
(X_FP[best_words].sum()/X_FP.shape[0]).plot(kind='bar',label="FP",color='g')
plt.margins(x=0, y=0)
plt.ylabel("mittlere Worthäufigkeit")
plt.ylim(0.01,60)
plt.yscale("log")

ax = plt.subplot(2, 2, 3)
(X_FN[best_words].sum()/X_FN.shape[0]).plot(kind='bar',label="FN",color='k')
plt.margins(x=0, y=0)
plt.ylabel("mittlere Worthäufigkeit")
plt.ylim(0.01,60)
plt.yscale("log")

plt.subplot(2, 2, 4)
(X_TP[best_words].sum()/X_TP.shape[0]).plot(kind='bar',label="TP",color='b')
plt.margins(x=0, y=0)
plt.ylabel("mittlere Worthäufigkeit")
plt.ylim(0.01,60)
plt.yscale("log")
    
plt.figtext(0.5, 1.09, r"Schätzung", {'fontsize': 30},
     horizontalalignment='center',
     verticalalignment='top')
plt.figtext(0.25, 1.02, r"Fake", {'fontsize': 20},
     horizontalalignment='center',
     verticalalignment='bottom',)
plt.figtext(0.75, 1.02, r"Real", {'fontsize': 20},
     horizontalalignment='center',
     verticalalignment='bottom',)
    
plt.figtext(-0.07, 0.5, r"Wahrheit", {'fontsize': 30},
     horizontalalignment='left',
     verticalalignment='center',
     rotation=90)
plt.figtext(0.00, 0.75, r"Fake", {'fontsize': 20},
     horizontalalignment='right',
     verticalalignment='center',)
plt.figtext(0.00, 0.25, r"Real", {'fontsize': 20},
     horizontalalignment='right',
     verticalalignment='center',)
    
plt.margins(x=0, y=0)
plt.tight_layout()
#plt.show()
plt.savefig("../build/plots/bow/500/cnfn_hist.pdf", bbox_inches = 'tight')
plt.close()<jupyter_output><empty_output><jupyter_text>RF VergleichsmodellTraining einer RF auf dem bow Input und Evaluierung<jupyter_code>RF = RandomForestClassifier(n_estimators=100, max_depth=10,random_state=0,criterion='entropy')
RF.fit(X_train.values,Y_train.label.values)
y_pred_bool_RF = RF.predict(X_test.values)
y_pred_RF = RF.predict_proba(X_test.values)
y_pred_RF = y_pred_RF[:,1]
y_pred_train_RF = RF.predict_proba(X_train.values)
y_pred_train_RF = y_pred_train_RF[:,1]
Y_test['prediction_RF'] = y_pred_RF
Y_test['prediction_bool_RF'] = y_pred_bool_RF
Y_train['prediction_RF'] = y_pred_train_RF
print(classification_report(Y_test.label, Y_test.prediction_bool_RF))

#Confusion Matrix
cnfn_matrix = pd.crosstab(Y_test.label, Y_test.prediction_bool_RF, rownames=['Wahrheit'], colnames=['Schätzung'])
print(cnfn_matrix)
cnfn_matrix.columns = ['Fake','Real']
cnfn_matrix = cnfn_matrix.rename_axis("Schätzung", axis="columns")
cnfn_matrix.rename(index = {0.0: "Fake", 1.0:'Real'}, inplace = True) 
cnfn_matrix = cnfn_matrix/Y_test.shape[0]
sn.heatmap(cnfn_matrix, annot=True , cmap='viridis')
#plt.show()
plt.savefig("../build/plots/bow/500/RF/cnfsn_mtx_bow_best_nn.pdf")
plt.close()

#Overtraining test
bin_edges = np.linspace(0,1,11)
plt.hist(Y_test.prediction_RF[Y_test.label == 0],label="fake test", alpha = 0.4, color = "r",density=True,bins=bin_edges)
plt.hist(Y_train.prediction_RF[Y_train.label == 0],label='fake train', alpha = 0.4, color = 'r', histtype='step',density=True,bins=bin_edges)
plt.hist(Y_test.prediction_RF[Y_test.label == 1],label = "real test",alpha = 0.4, color = "b",density=True,bins=bin_edges)
plt.hist(Y_train.prediction_RF[Y_train.label == 1],label='real train', alpha = 0.4, color = 'b', histtype='step',density=True,bins=bin_edges)

plt.xlabel("Geschätzte Likelihood")
plt.ylabel("Dichte")
plt.legend(loc='upper center')
#plt.show()
plt.savefig("../build/plots/bow/500/RF/prob_bow_best_nn.pdf")
plt.close()

fpr_RF = dict()
tpr_RF = dict()
roc_auc_RF = dict()
fpr_RF, tpr_RF, _ = roc_curve(Y_test.label, Y_test.prediction_RF)
roc_auc_RF = auc(fpr_RF, tpr_RF)

plt.figure()
lw = 2
plt.plot(fpr_RF, tpr_RF, color='darkorange',
         lw=lw, label='ROC Kurve (AUC = %0.2f)' % roc_auc_RF)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")
#plt.show()
plt.savefig("../build/plots/bow/500/RF/roc_Hyperopt_bow_best_nn.pdf")
plt.close()<jupyter_output>precision    recall  f1-score   support

           0       0.87      0.73      0.79      3658
           1       0.81      0.91      0.86      4713

   micro avg       0.83      0.83      0.83      8371
   macro avg       0.84      0.82      0.83      8371
weighted avg       0.84      0.83      0.83      8371

Schätzung     0     1
Wahrheit             
0          2679   979
1           405  4308<jupyter_text>ComparisonVergleich des Sequential mit dem RF in der ROC Curve<jupyter_code>plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
         lw=lw, label='Sequential (AUC = %0.2f)' % roc_auc)
plt.plot(fpr_RF, tpr_RF, color='darkred',
         lw=lw, label='RandomForest (AUC = %0.2f)' % roc_auc_RF)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")
#plt.show()
plt.savefig("../build/plots/bow/500/roc_comparison.pdf")
plt.close()<jupyter_output><empty_output><jupyter_text>Draw Modell<jupyter_code>from graphviz import Digraph

m = Digraph(name='Modell', node_attr={'shape': 'record'})
m.attr(rankdir='LR')

m.node("input", r"𝐄𝐢𝐧𝐠𝐚𝐧𝐠𝐬 𝐋𝐚𝐠𝐞 | 500 Wörter ")
m.node("first", r"𝐄𝐫𝐬𝐭𝐞 𝐋𝐚𝐠𝐞|Neuronen: 1770|Aktivierung: ReLu|L1: 0.00011")
m.node("second", r"𝐙𝐰𝐞𝐢𝐭𝐞 𝐋𝐚𝐠𝐞|Neuronen: 9|Aktivierung: ReLu|L2: 0.0272")
m.node("output", r"𝐀𝐮𝐬𝐠𝐚𝐧𝐠𝐬 𝐋𝐚𝐠𝐞|Neuronen: 1 |Aktivierung: Sigmoid")

m.edge('input', 'first')
m.edge('first', 'second')
m.edge('second', 'output')

m.render('../build/plots/bow/500/modell_scheme', view=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1525436540932799, 'fraction_numerical': 0.03760677299871495, 'mean_word_length': 3.757101761956131, 'pattern_counts': {'":': 5, '<': 71, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '534201', 'license': 'MIT', 'n_tokens_mistral': 10502, 'n_tokens_neox': 9717, 'n_words': 1464, 'path': 'python/Sequential_bow.ipynb', 'repo_name': 'Cl3V0r/MLSeminar', 'size': '26458'}