<jupyter_start><jupyter_text>Kaggle - WiDS Datathon 2022 Modelo Histórico - Lags + medias históricas  1. Modules<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.filterwarnings('ignore')

#model
from catboost import CatBoostRegressor

import xgboost
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error as MSE<jupyter_output><empty_output><jupyter_text>2. Data<jupyter_code>train_path = 'https://raw.githubusercontent.com/ccollado7/wds-datathon-2022/main/data/raw/train.csv'
test_path = 'https://raw.githubusercontent.com/ccollado7/wds-datathon-2022/main/data/raw/test.csv'
submit_path = 'https://raw.githubusercontent.com/ccollado7/wds-datathon-2022/main/data/raw/sample_solution.csv'<jupyter_output><empty_output><jupyter_text>Datos típicos:+ Se usan solo para referenciahttps://www.energystar.gov/sites/default/files/tools/Climate_and_Weather_2020_508.pdf+ Referencias de Site_EUI y Source_EUI  (eui_normal.pdf)https://www.energystar.gov/buildings/benchmark/understand_metrics/score_details<jupyter_code>ref_path= 'https://raw.githubusercontent.com/ccollado7/wds-datathon-2022/main/data/raw/eui_normal.csv'
#train dataset
train = pd.read_csv(train_path)
train.head()
#test dataset
test = pd.read_csv(test_path)
test.head()
submit = pd.read_csv(submit_path)
submit.head()<jupyter_output><empty_output><jupyter_text>Datos comparativos<jupyter_code>referencia = pd.read_csv(ref_path,sep='\t',header=None)
referencia.columns=['facility_type','site_eui_ref','source_eui_ref']
referencia.set_index('facility_type',inplace=True)
resumen=train.groupby(['facility_type'])['site_eui','energy_star_rating'].mean()
resumen2=test.groupby(['facility_type'])['energy_star_rating'].median()
tabla=pd.merge(referencia,resumen,left_index=True, right_index=True)[['site_eui_ref','site_eui','energy_star_rating','source_eui_ref']]
tabla['ratio']=tabla.site_eui_ref/tabla.site_eui*100
tabla
#pd.merge(referencia,resumen2,left_index=True, right_index=True)[['site_eui_ref','energy_star_rating']]<jupyter_output><empty_output><jupyter_text>Según referencias son importantes cooling_degree_days y heating_degree_days, CDD y HDD<jupyter_code>print('CDD - median:',train.cooling_degree_days.median())
print('HDD - median:',train.heating_degree_days.median())<jupyter_output>CDD - median: 1277.0
HDD - median: 4337.0<jupyter_text>Los valores medios de USA son CDD: 1364 y HDD: 4233  https://www.energystar.gov/sites/default/files/tools/Multifamily_August_2018_EN_508.pdf<jupyter_code>train.columns
facility=data.facility_type.unique()
facility<jupyter_output><empty_output><jupyter_text>Algunos outliers en temperaturas<jupyter_code>plt.plot(train.december_avg_temp)
plt.plot(train.january_avg_temp)
plt.plot(train.february_avg_temp)
plt.plot(train.march_avg_temp)
plt.plot(train.april_avg_temp)
plt.plot(train.may_avg_temp)
plt.plot(train.december_min_temp)
plt.plot(train.january_min_temp)
plt.plot(train.february_min_temp)
plt.plot(train.march_min_temp)
plt.plot(train.april_min_temp)
plt.plot(train.may_min_temp)
plt.plot(train.june_avg_temp)
plt.plot(train.july_avg_temp)
plt.plot(train.august_avg_temp)
plt.plot(train.september_avg_temp)
plt.plot(train.october_avg_temp)
plt.plot(train.november_avg_temp)
plt.plot(train.cooling_degree_days,'x')
plt.plot(train.heating_degree_days,'x')<jupyter_output><empty_output><jupyter_text>3. Feature Engineering+ Reemplazo valores outliers por nulos+ Armo categóricas para variables que quiero que sean tratadas así por catboost+ Armo Lags  Años de construcción en 0, si quiero corregir year_built=0 por nan<jupyter_code>#train[train['year_built'] == 0].year_built=np.nan
#test[test['year_built'] == 0].year_built=np.nan
#train[train['building_class'] == 'Commercial'].ELEVATION=np.nan
#test[test['building_class'] == 'Commercial'].ELEVATION=np.nan
#sns.lmplot( x='floor_area', y='ELEVATION', data=test, fit_reg=False, hue='building_class', legend=True)
def list_duplicates(seq):
   seen = set()
   seen_add = seen.add
   # adds all elements it doesn't know yet to seen and all other to seen_twice
   seen_twice = set( x for x in seq if x in seen or seen_add(x) )
   # turn the set into a list (as requested)
   return list( seen_twice )<jupyter_output><empty_output><jupyter_text>Data = train + test<jupyter_code>test.index=test.index+len(train)
data=pd.concat([train,test],axis=0)<jupyter_output><empty_output><jupyter_text>Corrijo valores de viento iguales a 1 y los reemplazo por nan<jupyter_code>feature_corrected='direction_peak_wind_speed'
data[feature_corrected]= data[feature_corrected].apply(lambda x: np.nan if x == 1 else x)
feature_corrected='max_wind_speed'
data[feature_corrected]= data[feature_corrected].apply(lambda x: np.nan if x == 1 else x)
feature_corrected='direction_max_wind_speed'
data[feature_corrected]= data[feature_corrected].apply(lambda x: np.nan if x == 1 else x)<jupyter_output><empty_output><jupyter_text>Armo categóricas de year_built y floor_area para catboost<jupyter_code>data.year_built=data.year_built.astype('Int64')
data.floor_area=np.ceil(data.floor_area).astype('Int64')

data.year_built=data.year_built.astype('str')
data.floor_area=data.floor_area.astype('str')<jupyter_output><empty_output><jupyter_text>Genero ID para armar lags<jupyter_code>#data['id2']= data[['floor_area','building_class','year_built','facility_type','State_Factor','ELEVATION']].apply(lambda x: '{}{}{}{}{}{}'.format(x[0],x[1],x[2],x[3],x[4],x[5]), axis=1)
data['id2']= data[['State_Factor','building_class','facility_type','year_built','floor_area']].apply(lambda x: '{}{}{}{}{}'.format(x[0],x[1],x[2],x[3],x[4]), axis=1)
#data['idb']= data[['State_Factor','building_class','facility_type','year_built']].apply(lambda x: '{}{}{}{}'.format(x[0],x[1],x[2],x[3]), axis=1)
data.groupby(['Year_Factor'])['id2'].describe()
data[data.Year_Factor<7].id2.value_counts().sort_values(ascending=False)
data[data.Year_Factor==7].id2.value_counts().sort_values(ascending=False)
data.id2.value_counts().sort_values(ascending=False)<jupyter_output><empty_output><jupyter_text>Elijo las columnas a ser usadas en los lags, los lags incluyen del 1 al 6 (si existen y sino se reempalzan por nan)<jupyter_code>def make_lag(data,feature,i):
    data[feature+'_lag'+str(i)] = (data.groupby('id2',as_index=False)
                          .apply(lambda x: x[feature].shift(1))
                          .reset_index(level=0, drop=True))
    return data
#columnas=['energy_star_rating', 'january_min_temp', 'january_avg_temp', 'january_max_temp','february_min_temp', 'february_avg_temp', 'february_max_temp','march_min_temp', 'march_avg_temp', 'march_max_temp', 'april_min_temp',
#'april_avg_temp', 'april_max_temp', 'may_min_temp', 'may_avg_temp','may_max_temp', 'june_min_temp', 'june_avg_temp', 'june_max_temp',
#'july_min_temp', 'july_avg_temp', 'july_max_temp', 'august_min_temp','august_avg_temp', 'august_max_temp', 'september_min_temp',
#'september_avg_temp', 'september_max_temp', 'october_min_temp','october_avg_temp', 'october_max_temp', 'november_min_temp',
#'november_avg_temp', 'november_max_temp', 'december_min_temp','december_avg_temp', 'december_max_temp', 'cooling_degree_days','heating_degree_days', 'precipitation_inches', 'snowfall_inches',
#'snowdepth_inches', 'avg_temp', 'days_below_30F', 'days_below_20F','days_below_10F', 'days_below_0F', 'days_above_80F', 'days_above_90F',
#'days_above_100F', 'days_above_110F', 'direction_max_wind_speed','direction_peak_wind_speed', 'max_wind_speed', 'days_with_fog']

columnas=['energy_star_rating','heating_degree_days','cooling_degree_days', 'site_eui']

for i in range(1,7):
    for columna in columnas:
        data[columna+'_lag'+str(i)]=np.nan<jupyter_output><empty_output><jupyter_text>Función que arma lags (se toma su tiempo)<jupyter_code>for i in range(7,0,-1):
    print('----------------------------------------------------')   
    print('Year_Factor_'+str(i))   
    df=data[(data['Year_Factor']==i)]
    t=df['id2']
    b=df['id2'].unique()
    print('ids distintos en el año: ',len(b))
    id_duplicados=list_duplicates(t)
    res=[]
    print('cantidad duplicados:',len(id_duplicados))
    [res.append(x) for x in b if x not in id_duplicados]
    print('ids unicos:',len(res))
    rowIndex = data[(data['id2'].isin(id_duplicados))&(data.Year_Factor==i)].index
    data.loc[rowIndex,['id2']]=data.loc[rowIndex,['id2']]+str(i)
    
    resIndex = data[(data['id2'].isin(res))&(data.Year_Factor==i)].index
        
    for j in range(7,0,-1):
        if (i>j):
            dfx=data[(data['Year_Factor']==j)]
            a=(dfx.id2.unique())
            comunes=(set(a) & set(res))
            print('año->',i,'con previo->',j,'ids comunes ->',len(comunes),'lag->',i-j)
            cIndex = data.loc[(data[data.id2.isin(comunes)].index)&(data[(data.Year_Factor==i) | (data.Year_Factor==j)].index)].index
            for columna in columnas:
                data.loc[cIndex]=make_lag(data.loc[cIndex],columna,i-j)
#rowIndex=data[data.Year_Factor==7].energy_star_rating.notnull().index
#rowIndex=data[data.energy_star_rating_lag1.notnull()].index
#data.loc[rowIndex][['Year_Factor','energy_star_rating','site_eui','ELEVATION','cooling_degree_days','heating_degree_days']]
data.isnull().sum()
data.columns # se inluyen lags<jupyter_output><empty_output><jupyter_text>Implemento "last" como la media de los lags disponibles de las columnas elegidas<jupyter_code>columnas_last=[]
for columna in columnas:
    data[columna+'_last']=data[[columna+'_lag1',columna+'_lag2',columna+'_lag3',columna+'_lag4',columna+'_lag5',columna+'_lag6']].mean(axis=1)
    columnas_last.append(columna+'_last')
columnas_last
plt.plot(data.heating_degree_days_last,'x')
plt.plot(data.cooling_degree_days_last,'o')<jupyter_output><empty_output><jupyter_text>Armados los lags e _hist separo el dataset de train y test (train_ampliado y test_ampliado)<jupyter_code>train_ampliado=data[data.Year_Factor<7]
test_ampliado=data[data.Year_Factor==7]
test_ampliado.index=test_ampliado.index-len(train_ampliado)<jupyter_output><empty_output><jupyter_text>4. Modelos  Selecciono features<jupyter_code>features_selected=list(train.columns)+columnas_last #no uso lags sino last
X_train=train_ampliado[features_selected]
X_train = X_train.drop(['site_eui','id'],axis=1)
cat_features = list(X_train.select_dtypes(include=['object']).columns)
print(cat_features)
y = train_ampliado.site_eui
y

X_test=test_ampliado
X_test=X_test[features_selected]
X_test = X_test.drop(['id','site_eui'], axis=1)<jupyter_output>['State_Factor', 'building_class', 'facility_type', 'floor_area', 'year_built']<jupyter_text>Modelo 1 - Sin CV, default params -> Kaggle score: 25.705<jupyter_code>model_1 = CatBoostRegressor(cat_features = cat_features,verbose = 100, max_ctr_complexity = 6) #
model_1.fit(X_train,y)
y_predict=model_1.predict(X_test)
y_predict[y_predict<0]=1
submit['site_eui'] = y_predict
submit.to_csv('model_1_last.csv',index=False)
df_feature_importance = pd.DataFrame(model_1.get_feature_importance(prettified=True))
plt.figure(figsize=(8, 5));
feature_plot= sns.barplot(x="Importances", y="Feature Id", data=df_feature_importance[:30],palette="cool");
plt.title('features importance');
#le = LabelEncoder()

#for i in X_train.columns:
#    if X_train[i].dtypes == 'object':
#        X_train[i] = le.fit_transform(X_train[i])
#        print(i)

#for i in X_test.columns:
#    if X_test[i].dtypes == 'object':
#        X_test[i] = le.fit_transform(X_test[i])
#        print(i)<jupyter_output><empty_output><jupyter_text>Modelo 2 - CV para manejar lags e históricos -> Kaggle Score: 29.448+  entreno en años < year y valido en year<jupyter_code>def get_model():
    cat_params = {
        'iterations': 10000,
        'learning_rate': 0.025,
        'od_wait': 1000,
        'depth': 8,
        #'task_type' : 'GPU',
        #'devices' : '0',
        'verbose' : 1000,
        "objective": "RMSE",
        "loss_function": "RMSE",
        "cat_features" : cat_features
    }
    model =  CatBoostRegressor(**cat_params)
    return model
from sklearn import model_selection
kfold = model_selection.KFold(7, shuffle=True, random_state=2022)
models = []
train_with_k_fold = False
apply_psudo_labeling = False


for year in range(2,6):
    train_indices=X_train[X_train.Year_Factor<=year].index
    val_indices=X_train[X_train.Year_Factor==year+1].index
    X_train_cv = X_train.iloc[train_indices]
    y_train_cv = y.iloc[train_indices]
    X_val_cv = X_train.iloc[val_indices]
    y_val_cv = y.iloc[val_indices]
    model = get_model()
    model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv))
    models.append(model)
y_predict=np.median([model.predict(X_test) for model in models], axis=0)
y_predict
submit['site_eui'] = y_predict
submit.to_csv('modelo_cv_last.csv',index=False)
submit.head()
df_feature_importance = pd.DataFrame(models[0].get_feature_importance(prettified=True))
plt.figure(figsize=(8, 5));
feature_plot= sns.barplot(x="Importances", y="Feature Id", data=df_feature_importance[:30],palette="cool");
plt.title('features importance');<jupyter_output><empty_output><jupyter_text>Modelo 3 - CV clásico shuffle (puede tener problemas con lags e históricos) -> Kaggle Score: 23.114<jupyter_code>from sklearn import model_selection
kfold = model_selection.KFold(5, shuffle=True, random_state=2022)
models = []
train_with_k_fold = False
apply_psudo_labeling = False


for train_indices, val_indices in kfold.split(X_train):
    X_train_cv = X_train.iloc[train_indices]
    y_train_cv = y.iloc[train_indices]
    X_val_cv = X_train.iloc[val_indices]
    y_val_cv = y.iloc[val_indices]
    model = get_model()
    model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv))
    models.append(model)
y_predict=np.mean([model.predict(X_test) for model in models], axis=0)
submit['site_eui'] = y_predict
submit.to_csv('modelo_cv_overfitting.csv',index=False)
submit.head()
#38.465<jupyter_output><empty_output><jupyter_text>Modelo 4 - CV-modificado -> Kaggle Score: 22.249+ Se hacen 3 CV sólo del año 6 (para que sea un 10% de train)+ Solo se valida con datos del año 6+ Los 3 usan los mismos datos del año 1 al 5<jupyter_code>X_train=train_ampliado[features_selected]

X_train6=X_train[X_train.Year_Factor==6]

X_train1_5=X_train[X_train.Year_Factor<6]

X_train = X_train.drop(['site_eui','id'],axis=1)
cat_features= list(X_train.select_dtypes(include=['object']).columns)
print(cat_features)
y = train_ampliado.site_eui
y
y6=X_train6.site_eui
y1_5=X_train1_5.site_eui
X_train6 = X_train6.drop(['site_eui','id'],axis=1)
#X_train6 = X_train6.drop(['site_eui','id','id2'],axis=1)
X_train1_5 = X_train1_5.drop(['site_eui','id'],axis=1)
#X_train1_5 = X_train1_5.drop(['site_eui','id','id2'],axis=1)
def get_model():
    cat_params = {
        'iterations': 10000,
        'learning_rate': 0.025,
        'od_wait': 1000,
        'depth': 8,
        #'task_type' : 'GPU',
        #'devices' : '0',
        'verbose' : 1000,
        "objective": "RMSE",
        "loss_function": "RMSE",
        "cat_features" : cat_features
    }
    model =  CatBoostRegressor(**cat_params)
    return model
from sklearn import model_selection
kfold = model_selection.KFold(3, shuffle=True, random_state=2022)
models = []

base_indice=train_indices1_5[-1]
for train_indices, val_indices in kfold.split(X_train6):
    X_train_cv6 = X_train6.iloc[train_indices]
    X_train_cv=pd.concat([X_train1_5,X_train_cv6])
    #print(len(X_train_cv))
    #print('len_indices',len(train_indices))
    #print('val',len(val_indices))
    y_train_cv = pd.concat([y1_5,y6.iloc[train_indices]])
    X_val_cv = X_train6.iloc[val_indices]
    y_val_cv = y6.iloc[val_indices]
    model = get_model()
    model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv))
    models.append(model)
X_test=test_ampliado
X_test=X_test[features_selected]
X_test = X_test.drop(['id','site_eui'], axis=1)

y_predict=np.mean([model.predict(X_test) for model in models], axis=0)
y_predict[y_predict<0]=1
submit['site_eui'] = y_predict
submit.to_csv('modelo_cv_3.csv',index=False)
submit.head()
#38.465<jupyter_output><empty_output><jupyter_text>Modelo 4.b -> Kaggle Score: 22.173+ igual al 4 pero median en lugar de mean para votación de los modelos<jupyter_code>X_test=test_ampliado
X_test=X_test[features_selected]
X_test = X_test.drop(['id','site_eui'], axis=1)

y_predictm=np.median([model.predict(X_test) for model in models], axis=0)
y_predictm[y_predictm<0]=1
submit['site_eui'] = y_predictm
submit.to_csv('modelo_cv_3_median.csv',index=False)
submit.head()
plt.plot(y_predict,y_predictm,'x')
np.corrcoef(y_predict,y_predictm)<jupyter_output><empty_output><jupyter_text>Modelo 5 - saco otras features del dataset  -> Kaggle score: 22.569+ saco temperaturas con detalle mensual+ no mejoro respecto de dejarlas<jupyter_code>features_selected=list(train.columns)+columnas_last #no uso lags sino last
unwanted={'january_min_temp',
 'january_avg_temp',
 'january_max_temp',
 'february_min_temp',
 'february_avg_temp',
 'february_max_temp',
 'march_min_temp',
 'march_avg_temp',
 'march_max_temp',
 'april_min_temp',
 'april_avg_temp',
 'april_max_temp',
 'may_min_temp',
 'may_avg_temp',
 'may_max_temp',
 'june_min_temp',
 'june_avg_temp',
 'june_max_temp',
 'july_min_temp',
 'july_avg_temp',
 'july_max_temp',
 'august_min_temp',
 'august_avg_temp',
 'august_max_temp',
 'september_min_temp',
 'september_avg_temp',
 'september_max_temp',
 'october_min_temp',
 'october_avg_temp',
 'october_max_temp',
 'november_min_temp',
 'november_avg_temp',
 'november_max_temp',
 'december_min_temp',
 'december_avg_temp',
 'december_max_temp'}

features_selected= [ele for ele in features_selected if ele not in unwanted]
X_train=train_ampliado[features_selected]

X_train6=X_train[X_train.Year_Factor==6]

X_train1_5=X_train[X_train.Year_Factor<6]

X_train = X_train.drop(['site_eui','id'],axis=1)
cat_features= list(X_train.select_dtypes(include=['object']).columns)
print(cat_features)
y = train_ampliado.site_eui
y
y6=X_train6.site_eui
y1_5=X_train1_5.site_eui
X_train6 = X_train6.drop(['site_eui','id'],axis=1)
X_train1_5 = X_train1_5.drop(['site_eui','id'],axis=1)
from sklearn import model_selection
kfold = model_selection.KFold(3, shuffle=True, random_state=2022)
models = []

base_indice=train_indices1_5[-1]
for train_indices, val_indices in kfold.split(X_train6):
    X_train_cv6 = X_train6.iloc[train_indices]
    X_train_cv=pd.concat([X_train1_5,X_train_cv6])
    y_train_cv = pd.concat([y1_5,y6.iloc[train_indices]])
    X_val_cv = X_train6.iloc[val_indices]
    y_val_cv = y6.iloc[val_indices]
    model = get_model()
    model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv))
    models.append(model)
X_test=test_ampliado
X_test=X_test[features_selected]
X_test = X_test.drop(['id','site_eui'], axis=1)

y_predict=np.median([model.predict(X_test) for model in models], axis=0)
y_predict[y_predict<0]=1
submit['site_eui'] = y_predict
submit.to_csv('modelo_cv_3_sintemp_median.csv',index=False)
submit.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1478631589892898, 'fraction_numerical': 0.02006862847041697, 'mean_word_length': 7.085329970575872, 'pattern_counts': {'":': 4, '<': 101, '<?xml version=': 0, '>': 102, 'https://': 7, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '573315', 'license': 'MIT', 'n_tokens_mistral': 7793, 'n_tokens_neox': 7488, 'n_words': 1061, 'path': 'notebooks/modelos/modelo_1/modelo_historico.ipynb', 'repo_name': 'ccollado7/wds-datathon-2022', 'size': '19234'}