<jupyter_start><jupyter_text>Importing Datasets<jupyter_code>path = 'drive/My Drive//Colab Notebooks//datasets//jets//'

df_train = pd.read_csv(path+'train_export_jets.csv',delimiter=',')
df_test = pd.read_csv(path+'test_export_jets.csv',delimiter=',')
df_validation = pd.read_csv(path+'val_export_jets.csv',delimiter=',')

df_train.drop(labels=['Unnamed: 0'],axis=1,inplace=True)
df_test.drop(labels=['Unnamed: 0'],axis=1,inplace=True)
df_validation.drop(labels=['Unnamed: 0'],axis=1,inplace=True)
def remove_zero_padded_features(df,padding = 5):
    df_feature_description = df.describe()
    df_filtered = pd.DataFrame()
    indexes = np.array(range(0,df_feature_description.shape[1],padding))

    for ii in indexes:
      cols = df.columns[ii:ii+padding]
      if  np.all(df_feature_description.loc['max',cols] == 0)\
      and np.all(df_feature_description.loc['min',cols] == 0):
        continue
      df_filtered[cols] = df[cols]
    return df_filtered
X_train = remove_zero_padded_features(df_train.drop(labels=['class','njets'],axis=1))
selected_columns = X_train.columns
X_test = df_test[selected_columns]
X_val = df_validation[selected_columns]

y_train = df_train['class'].astype(int).values
y_test = df_test['class'].astype(int).values
y_val = df_validation['class'].astype(int).values

X_train.shape<jupyter_output><empty_output><jupyter_text>Separating sets and Pre-processing<jupyter_code>X_train_no_anomaly = df_train[df_train['class'] < 1.0][selected_columns].copy()
X_train_anomaly = df_train[df_train['class'] == 1.0][selected_columns].copy()

X_val_no_anomaly = df_validation[df_validation['class'] < 1.0][selected_columns].copy()
X_val_anomaly = df_validation[df_validation['class'] == 1.0][selected_columns].copy()

X_test_no_anomaly = df_test[df_test['class'] < 1.0][selected_columns].copy()
X_test_anomaly = df_test[df_test['class'] == 1.0][selected_columns].copy()
scaler1 = MinMaxScaler(feature_range=(-1,1)).fit(X_train)

X_train_norm_no_anomaly = scaler1.transform(X_train_no_anomaly)
X_val_norm_no_anomaly = scaler1.transform(X_val_no_anomaly)
X_test_norm_no_anomaly = scaler1.transform(X_test_no_anomaly)

# scaler2 = StandardScaler().fit(X_train_anomaly)
# X_train_norm_anomaly = scaler2.transform(X_train_anomaly)
# X_val_norm_anomaly = scaler2.transform(X_val_anomaly)
# X_test_norm_anomaly = scaler2.transform(X_test_anomaly)

X_train_norm = scaler1.transform(X_train)
X_val_norm = scaler1.transform(X_val)
X_test_norm = scaler1.transform(X_test)<jupyter_output><empty_output><jupyter_text>Configuring AE<jupyter_code>def build_simple_autoencoder():
  nb_epoch = 500
  batch_size = 32
  input_dim = X_train_norm_no_anomaly.shape[1]
  encoding_dim = 55
  hidden_dim = int(encoding_dim / 2)
  learning_rate = 1e-1

  input_layer = Input(shape=(input_dim, ))

  #encoder
  encoder = Dense(encoding_dim)(input_layer)
  encoder = BatchNormalization()(encoder)
  encoder = Activation(activation='relu')(encoder)

  encoder = Dense(hidden_dim)(encoder)
  encoder = BatchNormalization()(encoder)
  encoder = Activation(activation='relu')(encoder)

  encoder = Dense(int(hidden_dim/2))(encoder)
  encoder = BatchNormalization()(encoder)
  encoder = Activation(activation='relu')(encoder)

  #decoder
  decoder = Dense(hidden_dim)(encoder)
  decoder = BatchNormalization()(decoder)
  decoder = Activation(activation='relu')(decoder)

  decoder = Dense(encoding_dim)(decoder)
  decoder = BatchNormalization()(decoder)
  decoder = Activation(activation='relu')(decoder)

  decoder = Dense(input_dim)(decoder)
  decoder = BatchNormalization()(decoder)
  decoder = Activation(activation='tanh')(decoder)

  autoencoder = Model(inputs=input_layer, outputs=decoder)
  autoencoder.summary()
  return autoencoder
autoencoder = build_simple_autoencoder()
autoencoder.compile(metrics=['accuracy'],
                    loss='mean_squared_error',
                    optimizer='adam')
cp = ModelCheckpoint(filepath="autoencoder_classifier_reconstruction.h5",
                               save_best_only=True,
                               verbose=0)

earlyStopping = EarlyStopping(monitor = 'val_loss', patience = 30, mode = 'auto')
start_time = time.time()
print('Starting Training for AutoEncoder...')
history = autoencoder.fit(X_train_norm_no_anomaly, X_train_norm_no_anomaly,
                    epochs=nb_epoch,
                    batch_size=batch_size,
                    shuffle=True,
                    validation_data=(X_val_norm_no_anomaly, X_val_norm_no_anomaly),
                    verbose=1,
                    callbacks=[cp,earlyStopping]).history
print('Total elapsed time:' ,(time.time() - start_time), 'seconds')
plt.style.use('seaborn-whitegrid')  
plt.figure(figsize=(12, 8))
plt.plot(history['loss'], linewidth=2, label='Train')
plt.plot(history['val_loss'], linewidth=2, label='Valid')
plt.legend(loc='upper right')
plt.title('Model loss - AutoEncoder for Reconstruction')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.grid(True)
plt.savefig('Model_loss_AutoEncoder_reconstruction.png')
plt.show()
def mean_squared_error(real,prediction):
  return np.mean(((real - prediction)**2), axis=1)

def rmse(real,prediction):
  return np.mean(((real - prediction)**2)**0.5, axis=1)
X_train_no_anomaly_pred = autoencoder.predict(X_train_norm_no_anomaly)

X_train_rmse_array = rmse(X_train_no_anomaly_pred,X_train_norm_no_anomaly)
X_train_rmse = np.mean(X_train_rmse_array)
print('rmse = ',X_train_rmse)
df_train_no_anomaly = pd.DataFrame(X_train_norm_no_anomaly,columns=selected_columns)
display(pd.DataFrame(X_train_norm_no_anomaly,columns=selected_columns).head())
display(pd.DataFrame(X_train_no_anomaly_pred,columns=selected_columns).head())
X_train_rmse_array.min()

X_predictions = autoencoder.predict(X_val_norm)
mse = mean_squared_error(X_val_norm, X_predictions)
error_df = pd.DataFrame({'Reconstruction_error': mse,
                        'True_class': y_val})
                        # 'True_class': y_train[y_train < 1.0]})

precision_rt, recall_rt, threshold_rt = precision_recall_curve(error_df.True_class, error_df.Reconstruction_error)
plt.style.use('seaborn-whitegrid')  
plt.figure(figsize=(14,8))
plt.plot(threshold_rt[:-1], precision_rt[1:-1], label="Precision",linewidth=5)
plt.plot(threshold_rt[:-1], recall_rt[1:-1], label="Recall",linewidth=5)
plt.title('Precision and recall for different MSE threshold values')
plt.xlabel('MSE Threshold')
plt.ylabel('Precision/Recall')
plt.legend()
plt.xlim(0,0.05)
plt.grid(True)
# plt.savefig('Precision_Recall_AutoEncoder_reconstruction.png')
X_test_pred = autoencoder.predict(X_test_norm)
y_pred_class = mean_squared_error(X_test_pred, X_test_norm).ravel()

threshold = 0.005
y_pred_class[y_pred_class > threshold] = 1
y_pred_class[y_pred_class <= threshold] = 0<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>df_prediction = pd.DataFrame({'Predicted': y_pred_class.ravel(),
                        'True_class': y_test.ravel()})

LABELS = ["Background","Signal"]
conf_matrix = confusion_matrix(df_prediction.True_class, df_prediction.Predicted)
plt.figure(figsize=(8, 8))
sns.heatmap(conf_matrix, xticklabels=LABELS, yticklabels=LABELS, annot=True, fmt="d");
plt.title("Confusion matrix")
plt.xlabel('True class')
plt.ylabel('Predicted class')
plt.show()
target_names = ["Background","Signal"]
c = classification_report(df_prediction.True_class, df_prediction.Predicted,target_names = target_names)
print(c)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13541806468930345, 'fraction_numerical': 0.012078915581801101, 'mean_word_length': 6.2631578947368425, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '573984', 'license': 'MIT', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2611, 'n_words': 329, 'path': 'AutoEncoder_Reconstruction.ipynb', 'repo_name': 'lucasthim/data-science', 'size': '7451'}