<jupyter_start><jupyter_text>Ready to feed the ANN<jupyter_code>xvals = [90, 180, 365, 730, 1095, 1460, 1825, 2190, 2555, 2920, 3285, 3650, 4015, 4380]

def loadData(trainingPercentage = 0.8):
    path_to_cleaned_data_SX5E = config.path_to_cleaned_data_SX5E
    path_to_cleaned_data_NKY = config.path_to_cleaned_data_NKY
    path_to_cleaned_data_SPX = config.path_to_cleaned_data_SPX
    path_to_cleaned_data_UKX = config.path_to_cleaned_data_UKX

    with open(path_to_cleaned_data_SX5E) as json_file:
        dictionary_SX5E = json.load(json_file)
    with open(path_to_cleaned_data_NKY) as json_file:
        dictionary_NKY = json.load(json_file)
    with open(path_to_cleaned_data_SPX) as json_file:
        dictionary_SPX = json.load(json_file)
    with open(path_to_cleaned_data_UKX) as json_file:
        dictionary_UKX = json.load(json_file)
    
    # Interpolating the data
    
    input_vector_SX5E = []
    input_vector_NKY = [] 
    input_vector_SPX = []
    input_vector_UKX = []
    input_vector_SX5E_orig = []
    input_vector_NKY_orig = [] 
    input_vector_SPX_orig = []
    input_vector_UKX_orig = []

    for key in dictionary_SX5E.keys():
        x = dictionary_SX5E[key][0]
        y = dictionary_SX5E[key][1]
        yinterp = np.interp(xvals, x, y)
        #computing new interpolated values
        dictionary_SX5E[key][0] = xvals
        dictionary_SX5E[key][1] = yinterp
        input_vector_SX5E.append(yinterp.tolist())
        input_vector_SX5E_orig.append(pd.Series(y,index=x))

    for key in dictionary_NKY.keys():
        x = dictionary_NKY[key][0]
        y = dictionary_NKY[key][1]
        yinterp = np.interp(xvals, x, y)
        #computing new interpolated values
        dictionary_NKY[key][0] = xvals
        dictionary_NKY[key][1] = yinterp
        input_vector_NKY.append(yinterp.tolist())
        input_vector_NKY_orig.append(pd.Series(y,index=x))

    for key in dictionary_SPX.keys():
        x = dictionary_SPX[key][0]
        y = dictionary_SPX[key][1]
        yinterp = np.interp(xvals, x, y)
        #computing new interpolated values
        dictionary_SPX[key][0] = xvals
        dictionary_SPX[key][1] = yinterp
        input_vector_SPX.append(yinterp.tolist())
        input_vector_SPX_orig.append(pd.Series(y,index=x))

    for key in dictionary_UKX.keys():
        x = dictionary_UKX[key][0]
        y = dictionary_UKX[key][1]
        yinterp = np.interp(xvals, x, y)
        #computing new interpolated values
        dictionary_UKX[key][0] = xvals
        dictionary_UKX[key][1] = yinterp
        input_vector_UKX.append(yinterp.tolist())
        input_vector_UKX_orig.append(pd.Series(y,index=x))
    
        
    # Computing the input_training_set and input_validation_set
    stop_NKY = int(len(input_vector_NKY)*trainingPercentage)
    stop_SPX = int(len(input_vector_SPX)*trainingPercentage)
    stop_SX5E = int(len(input_vector_SX5E)*trainingPercentage)
    stop_UKX= int(len(input_vector_UKX)*trainingPercentage)

    input_training_set_NKY = input_vector_NKY[:stop_NKY]
    input_validation_set_NKY = input_vector_NKY[stop_NKY:]
    input_training_set_SPX = input_vector_SPX[:stop_SPX]
    input_validation_set_SPX = input_vector_SPX[stop_SPX:]
    input_training_set_SX5E = input_vector_SX5E[:stop_SX5E]
    input_validation_set_SX5E = input_vector_SX5E[stop_SX5E:]
    input_training_set_UKX = input_vector_UKX[:stop_UKX]
    input_validation_set_UKX = input_vector_UKX[stop_UKX:]

    input_training_set_NKY_orig = input_vector_NKY_orig[:stop_NKY]
    input_validation_set_NKY_orig = input_vector_NKY_orig[stop_NKY:]
    input_training_set_SPX_orig = input_vector_SPX_orig[:stop_SPX]
    input_validation_set_SPX_orig = input_vector_SPX_orig[stop_SPX:]
    input_training_set_SX5E_orig = input_vector_SX5E_orig[:stop_SX5E]
    input_validation_set_SX5E_orig = input_vector_SX5E_orig[stop_SX5E:]
    input_training_set_UKX_orig = input_vector_UKX_orig[:stop_UKX]
    input_validation_set_UKX_orig = input_vector_UKX_orig[stop_UKX:]
    
    input_training_set = np.vstack((input_training_set_NKY,
                                    input_training_set_SPX,
                                    input_training_set_SX5E,
                                    input_training_set_UKX))
    
    input_validation_set = (input_validation_set_NKY +
                            input_validation_set_SPX +
                            input_validation_set_SX5E + 
                            input_validation_set_UKX)
    
    input_training_set_orig = (input_training_set_NKY_orig + 
                               input_training_set_SPX_orig + 
                               input_training_set_SX5E_orig + 
                               input_training_set_UKX_orig)
    
    input_validation_set_orig = (input_validation_set_NKY_orig + 
                                 input_validation_set_SPX_orig + 
                                 input_validation_set_SX5E_orig + 
                                 input_validation_set_UKX_orig)
    
    return input_training_set, input_validation_set, input_training_set_orig, input_validation_set_orig
#input_training_set,  input_testing_set = data_preprocessing.preprocessed_data_autoencoder_repo(0.8)
input_training_set,  input_testing_set, input_training_set_orig,  input_testing_set_orig = loadData(0.8)
scaler_train = MinMaxScaler()
df_train = pd.DataFrame(input_training_set).T
scaler_train.fit(df_train)
df_train = pd.DataFrame(scaler_train.transform(df_train)).T
input_training_set_normalized = np.asarray(df_train)
from sklearn.preprocessing import PolynomialFeatures

kf = KFold(n_splits=3, shuffle=True, random_state=42)

def MSE(y, ypred):
    size = y.size
    return np.sum((y-ypred)**2)/size

input_dim = input_training_set.shape[1]
autoencoder, encoder, decoder = models.autoencoder_model_repo(input_dim,l1_=0.0001)

x_train = input_training_set_normalized
y_train = input_training_set_normalized
e = 500
b = 50
losses = []
val_losses = []

for train_index, val_index in kf.split(x_train, y_train):
    x = x_train[train_index]
    y = y_train[train_index]
    hist = autoencoder.fit(x, y, epochs = e, batch_size = b, validation_data=(x_train[val_index], y_train[val_index]))
    losses.extend(hist.history['loss'])
    val_losses.extend(hist.history['val_loss'])
autoencoder.summary()
autoencoder.predict(np.ones((10,14)))
epochs = range(1500)
plt.plot(epochs, losses)
plt.plot(epochs, val_losses)

plt.title("CV Losses and Val_loss for Forwards")
plt.xlabel("epochs")
plt.ylabel("loss")<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code>scaler_test = MinMaxScaler()
df_test = pd.DataFrame(input_testing_set).T
scaler_test.fit(df_test)
df_test = pd.DataFrame(scaler_test.transform(df_test)).T
input_testing_set_normalized = np.asarray(df_test)
predicted_set_CV_normalized = autoencoder.predict(input_testing_set_normalized)
predicted_set_CV = np.asarray(pd.DataFrame(scaler_test.inverse_transform(pd.DataFrame(predicted_set_CV_normalized).T)).T)
import seaborn as sns 
MSE_CV = np.mean((predicted_set_CV-input_testing_set)**2, axis = 1)
sns.distplot(MSE_CV, bins = 70)
plt.xlim(0,np.max(MSE_CV))
plt.title('Cross Testing MSE for Repo')
autoencoder_json_CV = autoencoder.to_json()
encoder_json_CV = encoder.to_json()
decoder_json_CV = decoder.to_json()

with open("../project/model/saved_model/autoencoder_CV.json", "w") as json_file:
    json.dump(autoencoder_json_CV, json_file)

with open("../project/model/saved_model/encoder_CV.json", "w") as json_file:
    json.dump(encoder_json_CV, json_file)
    
with open("../project/model/saved_model/decoder_CV.json", "w") as json_file:
    json.dump(decoder_json_CV, json_file)

    
autoencoder.save('../project/model/saved_model/autoencoder_CV.h5')
encoder.save('../project/model/saved_model/encoder_CV.h5')
decoder.save('../project/model/saved_model/decoder_CV.h5')
input_testing_set_normalized.shape
predicted_set_CV_normalized.shape
plt.plot(predicted_set_CV_normalized[0])
plt.plot(input_testing_set_normalized[0])
plt.plot(np.mean((predicted_set_CV_normalized-input_testing_set_normalized)**2, axis = 1))
np.argmax(np.mean((predicted_set_CV_normalized-input_testing_set_normalized)**2, axis = 1))

scaler_test.fit(df_test)

threshold = 0.006
def scaleObservation(interpolatedObs, nonInterpolatedObs):
    commonMax = np.amax(interpolatedObs)#max(np.amax(interpolatedObs),nonInterpolatedObs.max()) 
    commonMin = np.amin(interpolatedObs)#min(np.amin(interpolatedObs),nonInterpolatedObs.min()) 
    return (nonInterpolatedObs - commonMin)/(commonMax - commonMin)

funcScaleObservation = lambda x : scaleObservation(x[0],x[1])
input_testing_set_orig_scaled = list(map(funcScaleObservation,
                                         list(zip(input_testing_set, input_testing_set_orig))))
def plotOutlier(threshold):
    indexList = np.where(np.mean((predicted_set_CV_normalized-input_testing_set_normalized)**2,axis = 1) > threshold)[0]
    unscaledValue = scaler_test.inverse_transform(input_testing_set_normalized)
    unscaledPredictedValue = scaler_test.inverse_transform(predicted_set_CV_normalized)
    for i in indexList:
        print(i)
        plt.plot(xvals, predicted_set_CV_normalized[i])
        plt.plot(xvals, input_testing_set_normalized[i],'-')
        plt.plot(input_testing_set_orig_scaled[i],'o')
        plt.show()
    return
plotOutlier(threshold)<jupyter_output>31<jupyter_text>Pas d'utilisation possible des données originales car le format change et l'autoencodeur attend un format fixe pour un nombre fixe d'input units. On ne peut comparer que l'interpolation et la prédiction de l'interpolation.On devrait alors se trouver avec une définition biaisée des outliers : l'interpolation donnera des observations bien différentes de celles observées.Bien que les points soient nombreux dans les graphes précédents, la courbe inteporlée ne passe absolument pas par les points et la détecte comme outlier. Ces observations seraient probablement considées comme inliers avec l'approche fonctionnelle (à vérifier).Faire attention à ne sélectionner qu'un marché précis.<jupyter_code>input_testing_set
input_testing_set_normalized[0]
predicted_set_CV_normalized[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09140839544919577, 'fraction_numerical': 0.015202040015692428, 'mean_word_length': 4.2806835836354225, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '55200', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3791, 'n_tokens_neox': 3624, 'n_words': 518, 'path': 'adam_api_repo_curve_anomaly_detection/notebooks/Training the Autoencoder.ipynb', 'repo_name': 'mChataign/smileCompletion', 'size': '10196'}