<jupyter_start><jupyter_text>Predicción de anomalías<jupyter_code>import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import warnings
warnings.simplefilter(action='ignore')

from keras.layers.core import Dense 
from keras.models import Model, Sequential
from keras import regularizers<jupyter_output>Using TensorFlow backend.<jupyter_text>Configuración del dataset  Balance adecuado entre las dos clases<jupyter_code>df = pd.read_pickle('sampledata2.pkl')

# Feature 'Problem': por cada día con 'ProblemReported' = 1, etiquetamos dicho día y los 2 días anteriores.
for device in df.DeviceID.unique():
    reported_array = df.loc[(df.DeviceID == device), 'ProblemReported'].values
    reported_array = reported_array[::-1]             # Recorreremos el vector al contrario, facilitando la implementación
    problem_array = np.zeros(len(reported_array))
    
    for i in range(len(reported_array)):
        if reported_array[i] == 1:
            
            for j in range(i, i + 3):             # En adelante al problema reportado, etiquetamos 2 días adicionales
                if j < len(reported_array):           # Evitamos el acceso a un índice fuera de los límites del array
                    problem_array[j] = 1
    
    problem_array = problem_array[::-1]
    df.loc[df.DeviceID == device, 'Problem'] = problem_array

df_problem_reported = df.loc[(df.Problem == 1)]
df_no_problem_reported = df.loc[(df.Problem == 0)]<jupyter_output><empty_output><jupyter_text>Selección de features<jupyter_code>features = ['Day', 'Month', 'Year', 'Problem']
features.extend([feature for feature in df.columns if 'Error' in feature and len(feature) == 15])
features.extend([feature for feature in df.columns if 'Grouped' in feature])<jupyter_output><empty_output><jupyter_text>B) Dataset para predicción de anomalías<jupyter_code># Reducción de registros para mantener un balance 90-10 (10% de anomalías, 90% sin anomalías) entre clases
np.random.seed(2019)
remove_n = df_no_problem_reported.shape[0] - (df_problem_reported.shape[0] * 9) # Cálculo del número de filas a borrar
drop_indices = np.random.choice(df_no_problem_reported.index, remove_n, replace=False)
df_no_problem_reported = df_no_problem_reported.drop(drop_indices)

df_B = pd.concat([df_no_problem_reported, df_problem_reported], ignore_index=True)
df_B.drop(columns=[feature for feature in df_B.columns if not feature in features], inplace=True)

df_B.shape<jupyter_output><empty_output><jupyter_text>Guardado del dataset para predicción de anomalías mediante modelos de clasificación sobre variables categóricas<jupyter_code>import pickle
import os

try:
    os.mkdir('Anomaly Prediction')
except:
    None

df_B.to_pickle('./Anomaly Prediction/cls_balanced_dataset.pkl')<jupyter_output><empty_output><jupyter_text>Ejecución de una batería de modelos de clasificación<jupyter_code>df_B = pd.read_pickle('./Anomaly Prediction/cls_balanced_dataset.pkl')
from sklearn.model_selection import train_test_split
from tpot import TPOTClassifier

X = df_B.iloc[:, :-1].values
y = df_B.iloc[:, -1].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=2019, shuffle=True, stratify=y)
tpot = TPOTClassifier(verbosity=3, scoring='accuracy', random_state=2019, periodic_checkpoint_folder='tpot_models', 
                      n_jobs=-1, generations=5, population_size=10, cv=3)

tpot.fit(X_train, y_train)
print(tpot.score(X_test, y_test))

tpot.export('./Anomaly Prediction/best_model_code.py')<jupyter_output>30 operators have been imported by TPOT.<jupyter_text>Predicción del conjunto de test y obtención de indicadores<jupyter_code>from sklearn.metrics import classification_report

y_pred = tpot.predict(X_test)

print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

        0.0       0.91      0.99      0.95     26424
        1.0       0.66      0.14      0.23      2936

avg / total       0.89      0.91      0.88     29360
{'fraction_non_alphanumeric': 0.09646061814556331, 'fraction_numerical': 0.023678963110667996, 'mean_word_length': 4.620448179271708, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '248764', 'license': 'MIT', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1352, 'n_words': 325, 'path': 'Notebooks/04 Anomaly Prediction - categorical features/TFM3b.ipynb', 'repo_name': 'debjitpal6/ML-based-PM', 'size': '4012'}