<jupyter_start><jupyter_text>MODELO PROYECTO IGAC<jupyter_code># Tratamiento de datos
# ==============================================================================
import numpy as np
import pandas as pd
import statsmodels.api as sm

# Gráficos
# ==============================================================================
import matplotlib.pyplot as plt

# Preprocesado y modelado
# ==============================================================================
from sklearn.datasets import load_boston
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import classification_report
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import ParameterGrid
from sklearn.inspection import permutation_importance
import multiprocessing


# Configuración warnings
# ==============================================================================
import warnings
warnings.filterwarnings('once')
#df= pd.read_("Proyecto_IGAC/data/DataFrame.pkl")
with open('../Datos_Limpios_modelo_2.csv') as f:
    df = pd.read_csv(f, delimiter=',')
df.head(30)
df.dtypes
#Eliminando columnas con base en el modelo 2 Y ESTANDARIZANDO LONGITUD Y LATITUD

#df['LONG_STD'] = (df['LONGITUD']-df['LONGITUD'].mean())/df['LONGITUD'].std()
#df['LAT_STD'] = (df['LATITUD']-df['LATITUD'].mean())/df['LATITUD'].std()

#df['H1_ESPESOR_STD'] = (df['H1_ESPESOR']-df['H1_ESPESOR'].mean())/df['H1_ESPESOR'].std()
#df['H1_RESULTADO_ph_STD'] = (df['H1_RESULTADO_ph']-df['H1_RESULTADO_ph'].mean())/df['H1_RESULTADO_ph'].std()
#df['H2_ESPESOR_STD'] = (df['H2_ESPESOR']-df['H2_ESPESOR'].mean())/df['H2_ESPESOR'].std()
#df['ALTITUD_STD'] = (df['ALTITUD']-df['ALTITUD'].mean())/df['ALTITUD'].std()
#df['PROFUNDIDAD_MAXIMA_STD'] = (df['PROFUNDIDAD MAXIMA']-df['PROFUNDIDAD MAXIMA'].mean())/df['PROFUNDIDAD MAXIMA'].std()



X.dtypes
#new_df['SEGUNDOS_N'].str.replace(',','.').astype(float).head(5)

#Specifiying features and the response variable
# ==============================================================================
df["ORDEN"] = df["ORDEN"].astype('category')
df["ORDEN_cat"] = df["ORDEN"].cat.codes
y = df['ORDEN_cat']


X.head(10)




y
cat_cols = X.select_dtypes(include=['object', 'category','bool']).columns.to_list()
cont_cols = numeric_cols = df.select_dtypes(include=['float64', 'int']).columns.to_list()

#CREATING DUMMIES FOR CATEGORICAL VARIABLES IN X MATRIX
# ==============================================================================
X= pd.get_dummies(X,  columns = cat_cols)
#X_cont = X[X[cont_cols]]

X.head(10)





#CREATING X AN Y FOR TRAINING AND TEST
# ==============================================================================
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state = 133)

#Para Francisco en R
list(X_train.index)
with open("X_train.csv","w") as f:
    for i in list(X_train.index):
        f.write(str(i)+",")
    f.close()

list(y_train.index)
with open("y_train.csv","w") as f:
    for i in list(y_train.index):
        f.write(str(i)+",")
    f.close()
    
list(y_test.index)
with open("y_test.csv","w") as f:
    for i in list(y_test.index):
        f.write(str(i)+",")
    f.close()
    
list(X_test.index)
with open("X_test.csv","w") as f:
    for i in list(X_test.index):
        f.write(str(i)+",")
    f.close()
y_train.value_counts()
y.value_counts()
len(y_test)
len(y_train)
from sklearn.utils.class_weight import compute_class_weight
weights= compute_class_weight('balanced',y_train.unique(),y_train)
y_train.unique()
weights
weight_dict={}
for i,j in enumerate(y_train.unique()):
    weight_dict[j]=weights[i]

weight_dict
# Basado en https://www.cienciadedatos.net/documentos/py08_random_forest_python.html
#https://www.analyticsvidhya.com/blog/2015/06/tuning-random-forest-model/ to estrategies for max features
# Grid de hiperparámetros evaluados
# ==============================================================================
param_grid = ParameterGrid(
                {'n_estimators': [150],
                 'class_weight' : ['balanced', 'balanced_subsample', weight_dict],
                 'max_features': ['auto', 'sqrt', 'log2'],
                 'max_depth'   : [None, 5, 15, 21, 31],
                 'criterion'   : ['gini', 'entropy']
                }
            )

# Loop para ajustar un modelo con cada combinación de hiperparámetros
# ==============================================================================
resultados = {'params': [], 'oob_accuracy': []}

for params in param_grid:
    
    modelo = RandomForestClassifier(
                oob_score    = True,
                n_jobs       = -1,
                random_state = 123,
                ** params
             )
    
    modelo.fit(X_train, y_train)
    
    resultados['params'].append(params)
    resultados['oob_accuracy'].append(modelo.oob_score_)
    print(f"Modelo: {params} \u2713")

# Resultados
# ==============================================================================
resultados = pd.DataFrame(resultados)
resultados = pd.concat([resultados, resultados['params'].apply(pd.Series)], axis=1)
resultados = resultados.sort_values('oob_accuracy', ascending=False)
resultados = resultados.drop(columns = 'params')
resultados.head(4)
# Mejores hiperparámetros por out-of-bag error
# ==============================================================================
print("--------------------------------------------------")
print("Mejores hiperparámetros encontrados (oob-accuracy)")
print("--------------------------------------------------")
print(resultados.iloc[0,0], ":", resultados.iloc[0,:]['oob_accuracy'], "accuracy")
resultados.iloc[0]
param_grid = ParameterGrid(
                {'n_estimators': [150],
                 'weight_class': ['balanced_subsample'],
                 'max_features': ['log2'],
                 'max_depth'   : [21],
                 'criterion'   : ['entropy']
                }
            )

mejor_modelo = RandomForestClassifier(
                oob_score    = True,
                n_jobs       = -1,
                random_state = 123,
                n_estimators = 150,
                class_weight = weight_dict,
                max_features = 'log2',
                max_depth = 31,
                criterion = 'entropy'
                
             )
    
mejor_modelo.fit(X_train, y_train)

mejor_modelo.feature_importances_
plt.figure(figsize=(15,70))
plt.barh(X.columns, mejor_modelo.feature_importances_)
# Probando el modelo
# ==============================================================================
pred = mejor_modelo.predict(X = X_test)
pred[:10]
X
X_test
#Matriz de confusión
#===================================================================================

mat_confusion = confusion_matrix(
                    y_true    = y_test,
                    y_pred    = pred
                )

accuracy = accuracy_score(
            y_true    = y_test,
            y_pred    = pred,
            normalize = True
           )

print("Matriz de confusión")
print("-------------------")
print(mat_confusion)
print("")
print(f"El accuracy de test es: {100 * accuracy} %")
categorias = df['ORDEN'].astype('category')

dic_cat = dict(enumerate(categorias.cat.categories))
print (dic_cat)
print(
    classification_report(
        y_true = y_test,
        y_pred = pred
    )
)<jupyter_output>precision    recall  f1-score   support

           0       1.00      0.99      0.99       902
           1       0.73      0.47      0.57       140
           2       1.00      0.96      0.98        26
           3       0.91      0.97      0.94      1023
           4       0.97      0.92      0.94       173

    accuracy                           0.94      2264
   macro avg       0.92      0.86      0.89      2264
weighted avg       0.94      0.94      0.94      2264<jupyter_text>TAREA DE JOSE Exportar el modelo y organizar el output para dejar todo en un pipeline que sea reproducible<jupyter_code>import pickle as pk
with open("CATS.pk","wb") as f:
    pk.dump(mejor_modelo,f)

mis_columnas=['H1_ESPESOR',
 'H1_RESULTADO_ph',
 'H2_ESPESOR',
 'ALTITUD',
 'PROFUNDIDAD_MAXIMA',
 'DRENAJE_NATURAL_Bien_drenado',
 'DRENAJE_NATURAL_Excesivo',
 'DRENAJE_NATURAL_Imperfecto',
 'DRENAJE_NATURAL_Moderadamente_bien_drenado',
 'DRENAJE_NATURAL_Moderadamente_excesivo',
 'DRENAJE_NATURAL_Moderado',
 'DRENAJE_NATURAL_Muy_pobre',
 'DRENAJE_NATURAL_Pobre',
 'EPIPEDON_Cambico',
 'EPIPEDON_Folistico',
 'EPIPEDON_Histico',
 'EPIPEDON_Melanico',
 'EPIPEDON_Molico',
 'EPIPEDON_Ocrico',
 'EPIPEDON_Umbrico',
 'FAMILIA_TEXTURAL_Arcillosa',
 'FAMILIA_TEXTURAL_Arenosa',
 'FAMILIA_TEXTURAL_Disica',
 'FAMILIA_TEXTURAL_Esqueletica',
 'FAMILIA_TEXTURAL_Euica',
 'FAMILIA_TEXTURAL_Fina',
 'FAMILIA_TEXTURAL_Fragmental',
 'FAMILIA_TEXTURAL_Franca',
 'FAMILIA_TEXTURAL_Gruesa_sobre_fragmental',
 'FAMILIA_TEXTURAL_Hidrosa',
 'FAMILIA_TEXTURAL_Limosa',
 'FAMILIA_TEXTURAL_Medial',
 'CONTENIDO_CENIZA_VOLCANICA_False',
 'CONTENIDO_CENIZA_VOLCANICA_True']
df=pd.DataFrame([[0.0 for i in range(len(mis_columnas)) ]],columns=mis_columnas)
lo_que_llega={"DRENAJE_NATURAL":'Imperfecto',"EPIPEDON":"Folistico", 'FAMILIA_TEXTURAL':'Fragmental', 'CONTENIDO_CENIZA_VOLCANICA': True, 
              'H1_ESPESOR':4, 'H1_RESULTADO_ph':6.3, 'H2_ESPESOR':26, 'ALTITUD':2600.9067, 'PROFUNDIDAD_MAXIMA':40.23 }

df["DRENAJE_NATURAL_"+lo_que_llega["DRENAJE_NATURAL"]]=1.0
df["EPIPEDON_"+lo_que_llega["EPIPEDON"]]=1.0
df["FAMILIA_TEXTURAL_"+lo_que_llega["FAMILIA_TEXTURAL"]]=1.0
df["CONTENIDO_CENIZA_VOLCANICA_"+str(lo_que_llega["CONTENIDO_CENIZA_VOLCANICA"])]=1.0


df["H1_ESPESOR"]=lo_que_llega["H1_ESPESOR"]
df['H1_RESULTADO_ph']=lo_que_llega['H1_RESULTADO_ph']
df['H2_ESPESOR']=lo_que_llega['H2_ESPESOR']
df['ALTITUD']=lo_que_llega['ALTITUD']
df['PROFUNDIDAD_MAXIMA']=lo_que_llega['PROFUNDIDAD_MAXIMA']<jupyter_output><empty_output><jupyter_text>No es una buena solucion hacerlo de esta forma, es mejor que hagamos un preprocesameinto que tenga en cuenta la estandarización y la creacion de las variables categoricas  Nuevo modelo ( cambiar el pipeline)<jupyter_code>from sklearn import preprocessing

from sklearn.compose import ColumnTransformer
from sklearn.datasets import fetch_openml
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.compose import make_column_selector
from sklearn.utils.class_weight import compute_class_weight
with open('../Datos_Limpios_modelo_2.csv') as f:
    df = pd.read_csv(f, delimiter=',')
df["CONTENIDO_CENIZA_VOLCANICA"]=df["CONTENIDO_CENIZA_VOLCANICA"].astype(str)
X = df.drop(columns=['ORDEN','IGUALDAD','H1_NOMENCLATURA','H1_COLOR_MATRIZ_HUMEDO1','H2_NOMENCLATURA','REGIMEN_HUMEDAD',
                  'CLASE_PENDIENTE','REGIMEN_TEMPERATURA','CLIMA_AMBIENTAL','TIPO_OBSERVACION','LONGITUD','LATITUD'])
X=X[sorted(X.columns)]

le = preprocessing.LabelEncoder()
y=le.fit_transform(df.ORDEN)

#numeric_features=X.select_dtypes(exclude=['object', 'category','bool']).columns.to_list()
#categorical_features=X.select_dtypes(include=['object', 'category','bool']).columns.to_list()
numeric_transformer = StandardScaler()
categorical_transformer = OneHotEncoder(handle_unknown='ignore')

preprocessor = ColumnTransformer(
    transformers=[
        ('categorical', categorical_transformer, make_column_selector(dtype_include=['object', 'category','bool'])),
        ('numeric',numeric_transformer,make_column_selector(dtype_exclude=['object', 'category','bool']))])


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, stratify=y, random_state = 133)
weights= compute_class_weight('balanced',classes=np.unique(y_train),y=y_train)

weight_dict={}
for i,j in enumerate(np.unique(y_train)):
    weight_dict[j]=weights[i]
    
    
mejor_modelo = RandomForestClassifier(
                oob_score    = True,
                n_jobs       = -1,
                random_state = 123,
                n_estimators = 150,
                class_weight = weight_dict,
                max_features = 'log2',
                max_depth = 31,
                criterion = 'entropy')
clf = Pipeline(steps=[('preprocessor', preprocessor),
                      ('classifier', mejor_modelo)])


clf.fit(X_train,y_train)

clf.score(X_test,y_test)<jupyter_output><empty_output><jupyter_text>Es importante que las columnas etén en el mismo orden en el que se entrenó el modelo para que todo sea reproducible<jupyter_code>a=X_test.iloc[0,np.random.choice(range(X_test.shape[1]),size=X_test.shape[1],replace=False)].to_dict()
new_a=dict([(key,a[key]) for key in sorted(a)])
prediction=clf.predict(pd.DataFrame(data=new_a,index=[0]))
import pickle as pk
with open("CATS_pipeline.pk","wb") as f:
    pk.dump(clf,f)
with open("CATS_label_encoder.pk","wb") as f:
    pk.dump(le,f)<jupyter_output><empty_output><jupyter_text>Pruebas para revisar que el moedlo se esté exportando y cargando bien<jupyter_code>with open("CATS_pipeline.pk", "rb") as f:
    clasifier=pk.load(f)
clasifier.score(X_test,y_test)
clf.predict(X_test.iloc[0].to_frame().T)
a=X_test.iloc[0,np.random.choice(range(X_test.shape[1]),size=X_test.shape[1],replace=False)].to_dict()
new_a=dict([(key,a[key]) for key in sorted(a)])
prediction=int(clf.predict(pd.DataFrame(data=new_a,index=[0]))[0])
result=X_test.iloc[:3,np.random.choice(range(X_test.shape[1]),size=X_test.shape[1],replace=False)].to_dict(orient="list")
new_result=dict([(key,result[key]) for key in sorted(result)])
columns=le.inverse_transform(clf.classes_.T)<jupyter_output><empty_output><jupyter_text>Exportar los datos de prueba para realizar predicciones en la aplicación<jupyter_code>with open("test_data.pk","wb") as f:
    pk.dump((X_test,y_test),f)
with open("test_data.pk","rb") as f:
    x_1,y_1=pk.load(f)<jupyter_output><empty_output><jupyter_text>Pruebas API  API predict<jupyter_code>import requests
import json
url="http://0.0.0.0:8000/api/predict"
data=X_test.iloc[2].to_dict()
x=requests.post(url,data=json.dumps(data))
result=dict(x.json())
result
result["prediction"]<jupyter_output><empty_output><jupyter_text>Si funciona!  API predict many<jupyter_code>X_test.shape
url='http://0.0.0.0:8000/api/predict_many'
data=X_test.iloc[:].to_dict(orient="list")
x=requests.post(url,data=json.dumps(data))
pd.Series(x.json()["predictions"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20212265735205212, 'fraction_numerical': 0.01853966547994895, 'mean_word_length': 4.530460624071322, 'pattern_counts': {'":': 3, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '171576', 'license': 'MIT', 'n_tokens_mistral': 5582, 'n_tokens_neox': 5228, 'n_words': 824, 'path': 'modelo_4_PROYECTO_IGAC.ipynb', 'repo_name': 'DS4A-Team19-2021/Agustin-Codazzi-Project', 'size': '14887'}