<jupyter_start><jupyter_text>JFK flights  Import de las librerias<jupyter_code>import numpy as np  
import pandas as pd 

#visualizacion de datos

import matplotlib.pyplot as plt 
import seaborn as sns 

#import graphviz

plt.style.use('ggplot')
pd.set_option('display.max_rows',100)
from sklearn import preprocessing<jupyter_output><empty_output><jupyter_text>Import de los datos<jupyter_code>url1 = 'https://bit.ly/3bf6QNP'
url2 = 'https://bit.ly/3kLR2p3'
df1 = pd.read_csv(url1)
df1.head()
df2 = pd.read_csv(url2)
df2.head()
df = df1.append(df2)
df.head()
df.drop('Unnamed: 0', axis = 1, inplace = True)<jupyter_output><empty_output><jupyter_text>Machine Learning Checklist - Analisis Univariante 2.1  Tamaño del dataset<jupyter_code>df.info(verbose = False)
#reset index
df.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>Visualizacion directa<jupyter_code>df.columns
df.head(1)<jupyter_output><empty_output><jupyter_text>Tipos de atributos<jupyter_code>df.info(verbose=True)
TARGET = 'ARR_DEL15'<jupyter_output><empty_output><jupyter_text>Estadísticos descriptivos<jupyter_code>df.describe().T
df.describe(exclude = np.number).T<jupyter_output><empty_output><jupyter_text>Valores nulos<jupyter_code>df.select_dtypes(include = np.number).isnull().sum()
df.select_dtypes(exclude = np.number).isnull().sum()<jupyter_output><empty_output><jupyter_text>Variable target - distribución<jupyter_code>df[TARGET].value_counts()
df[TARGET].mean()<jupyter_output><empty_output><jupyter_text>ML 2.1.7 - 11: EDA  Analisis de fechas<jupyter_code>df_LAX = df[df['ORIGIN'] == 'LAX']
df_LAX.head()
#FORMATO SQL
select
    FL_DATE,
    AVG(TARGET)
    FROM DF_LAX
    GROUP BY FL_DATE
df_LAX_dia = df_LAX[['FL_DATE',TARGET]].groupby(['FL_DATE']).mean()
df_LAX_dia.head()
df_LAX_dia.index = pd.to_datetime(df_LAX_dia.index)
df_LAX_dia.head()
df_LAX_dia[TARGET].plot()
df_LAX_dia[TARGET].resample('W').mean().plot()<jupyter_output><empty_output><jupyter_text>Inicio clase 2  Analisis de variables numéricas<jupyter_code>df['DISTANCE'].plot(kind = 'hist')
sns.boxplot(data = df, x= TARGET, y= 'DISTANCE')
sns.boxplot(data = df, x= TARGET, y= 'CRS_DEP_TIME')<jupyter_output><empty_output><jupyter_text>Analisis categoria<jupyter_code>df.pivot_table(index='ORIGIN_STATE_NM', values = TARGET, aggfunc = [len,sum,np.mean]).sort_values(by=[('mean','ARR_DEL15')], ascending = False)<jupyter_output><empty_output><jupyter_text>Data preparation  Ajuste de tipos<jupyter_code>df['AEROLINEA'] = df['OP_CARRIER_AIRLINE_ID'].astype(object)
df.drop('OP_CARRIER_AIRLINE_ID', axis =1, inplace= True)<jupyter_output><empty_output><jupyter_text>Nulos en numéricas<jupyter_code>df.select_dtypes(include = np.number).isnull().sum()
df['TAXI_OUT'].fillna(-1,inplace = True)
df.select_dtypes(include = np.number).isnull().sum()<jupyter_output><empty_output><jupyter_text>Nulos en categóricas<jupyter_code>df.select_dtypes(exclude = np.number).isnull().sum()
df['TAIL_NUM'].fillna('Sin matrícula',inplace = True)
df['AEROLINEA'].fillna('Sin aerolinea', inplace = True) #mismo que el 'otros'
df.select_dtypes(exclude = np.number).isnull().sum()<jupyter_output><empty_output><jupyter_text>Hemos imputado nulos de numéricas con valor extremo (fuera de la distribucion) y categoricas con una nueva categoría  Eliminacion de atributos sin varianza<jupyter_code>non_variance_column = ['DEST','DEST_CITY_NAME','DEST_STATE_NM']
df.drop(non_variance_column, axis = 1, inplace = True)
### Fechas - Sacar información
df['FL_DATE'].dtypes
df['FL_DATE'] = pd.to_datetime(df['FL_DATE'])
df['Year'] = df['FL_DATE'].dt.year
df['Month'] = df['FL_DATE'].dt.month
df['Day'] =  df['FL_DATE'].dt.day
df['Weekday'] =  df['FL_DATE'].dt.weekday
df['Week'] =  df['FL_DATE'].dt.week
df['Season'] = df['FL_DATE'].dt.quarter
df.head(1)
df.drop('FL_DATE', axis = 1, inplace = True)<jupyter_output><empty_output><jupyter_text>Transformacion de atributos<jupyter_code>df.select_dtypes(exclude = np.number).describe().T
# Frequency encoding para tail number
df_tail_num = pd.DataFrame(df['TAIL_NUM'].value_counts(dropna = False))
df_tail_num.head()
df_tail_num.columns = ['TAIL_NUM_COUNT']
df_tail_num['TAIL_NUM'] = df_tail_num.index
df_tail_num.head(10)
df = df.merge(df_tail_num, on = 'TAIL_NUM')
df.drop('TAIL_NUM', axis = 1, inplace= True)
del df_tail_num
df.head(1)<jupyter_output><empty_output><jupyter_text>OHE sobre el resto de categóricas<jupyter_code># ORIGIN
dummy_dataset = pd.get_dummies(df['ORIGIN'], prefix = 'ORIGIN')
dummy_dataset.head()
df = pd.concat([df,dummy_dataset], axis = 1)
df.drop('ORIGIN',axis = 1, inplace = True)
del dummy_dataset
def OHE(dataframe, column_name):

    dummy_dataset = pd.get_dummies(dataframe[column_name], prefix = column_name)
    dataframe = pd.concat([dataframe,dummy_dataset], axis = 1)
    dataframe.drop(column_name,axis = 1, inplace = True)
    del dummy_dataset

    return dataframe

#ORIGIN_CITY_NAME
df = OHE(df,'ORIGIN_CITY_NAME')
#ORIGIN_STATE_NM
df = OHE(df,'ORIGIN_STATE_NM')
#AEROLINEA
df = OHE(df,'AEROLINEA')
df.head().T<jupyter_output><empty_output><jupyter_text>ML Checklist 4. Modelling<jupyter_code>from sklearn import model_selection
from sklearn import metrics
model_selection??
#particion de validacion

# registros mas recientes

# aleatorio --> porque todos los meses deben estar representados
df.pivot_table(index= ['Year','Month'], values = TARGET, aggfunc = [len, sum, np.mean])
dev_df = df[(df['Year'] == 2018) | (df['Month'] <= 6)]
val_df = df[(df['Year'] == 2019) & (df['Month'] > 6)]
dev_df.shape
val_df.shape
dev_df_X = dev_df.drop(TARGET, axis = 1)
dev_df_y = dev_df[[TARGET]]
val_df_X = val_df.drop(TARGET, axis = 1)
val_df_y = val_df[[TARGET]]
# Split del development en train y test
model_selection.train_test_split?
X_train, X_test, y_train, y_test = model_selection.train_test_split(dev_df_X, dev_df_y, test_size=0.2, random_state=42)
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
import graphviz<jupyter_output><empty_output><jupyter_text>Pasos para lanzar un modelo * Instanciar el modelo: escoger el algoritmo y setear los hiperparametros* Entrenar el modelo: train set el algoritmo aprende las reglas* Predecir: test set* Evaluar: comparar la prediccion que tengo en test con el valor real del test  Instanciar el modelo<jupyter_code>dt = DecisionTreeClassifier(

                 #criterion="gini",
                 #splitter="best",
                 max_depth= 4, # número de preguntas que realiza el algoritmo
                 #min_samples_split=2, # numero de obs minimas en cada particion
                 min_samples_leaf=50,
                 #min_weight_fraction_leaf=0.,
                 #max_features=None, #numero de variables a evaluar en cada iteracion
                 random_state=42
                 #max_leaf_nodes=None,
                 #min_impurity_decrease=0.,
                 #min_impurity_split=None,
                 #class_weight=None,
                 #presort='deprecated'
)<jupyter_output><empty_output><jupyter_text>Entrenar el Modelo<jupyter_code>dt.fit(X_train,y_train)
# revisando las reglas que ha generado el modelo
dot_data = export_graphviz(decision_tree=dt,
                out_file=None,
                feature_names= X_test.columns,
                class_names=['Not delay','Delay']
                )
graphviz.Source(dot_data)<jupyter_output><empty_output><jupyter_text>Prediccion del modelo<jupyter_code>dt.predict(X_test)
dt.predict_proba(X_test)
y_test_pred = pd.DataFrame(dt.predict(X_test), index= y_test.index, columns = ['DelayPrediction'])
y_test_pred.head()
results_df = y_test.join(y_test_pred)
results_df['Success'] = (results_df['DelayPrediction'] == results_df[TARGET]).astype(int)
results_df.head()
#evaluar el modelo
#accuracy
results_df['Success'].mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13359703968355238, 'fraction_numerical': 0.009059589128493046, 'mean_word_length': 6.074007220216607, 'pattern_counts': {'":': 0, '<': 86, '<?xml version=': 0, '>': 87, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '550575', 'license': 'MIT', 'n_tokens_mistral': 3078, 'n_tokens_neox': 2975, 'n_words': 503, 'path': 'JFK_FLIGHTS_al_final_clase3.ipynb', 'repo_name': 'alfredo-bb/Nuclio-proyects', 'size': '7837'}