<jupyter_start><jupyter_text>Lectura de archivos<jupyter_code>%matplotlib inline
from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
id = '15eBo8goFc4q--lIGREc9Dknn9GqBybrs'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('test_values_selected_features_remix.csv')
test_values1 = pd.read_csv('test_values_selected_features_remix.csv', encoding='latin-1', index_col='building_id')
test_values1[test_values1.select_dtypes('O').columns] = test_values1[test_values1.select_dtypes('O').columns].astype('category')
id = '1F6ZTDoJc-aaiD-rpriXq5zmv2i84OSR4'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('train_values_selected_features_remix.csv')
train_values1 = pd.read_csv('train_values_selected_features_remix.csv', encoding='latin-1', index_col='building_id')
train_values1[train_values1.select_dtypes('O').columns] = train_values1[train_values1.select_dtypes('O').columns].astype('category')
id = '1kVPCXPedaMcZZIuj2wFjhJoQ36nD5kbq'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('train_values_complete_features_remix.csv')
train_values2 = pd.read_csv('train_values_complete_features_remix.csv', encoding='latin-1', index_col='building_id')
train_values2[train_values2.select_dtypes('O').columns] = train_values2[train_values2.select_dtypes('O').columns].astype('category')
id = '1XBWMAHR6sItnHT2dq3hX2nk9WrzNSF7J'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('test_values_complete_features_remix.csv')
test_values2 = pd.read_csv('test_values_complete_features_remix.csv', encoding='latin-1', index_col='building_id')
test_values2[test_values2.select_dtypes('O').columns] = test_values2[test_values2.select_dtypes('O').columns].astype('category')
id='1RUtolRcQlR3RGULttM4ZoQaK_Ouow4gc'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('train_labels.csv')
train_labels = pd.read_csv('train_labels.csv', encoding='latin-1', dtype={'building_id': 'int64', 'damage_grade': 'int64'}, index_col='building_id')
id = '1JDpXEXjP0QCLF7qbpMgWRdNCf5w3TURK'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('index.csv')

df_index = pd.read_csv('index.csv', encoding='latin-1', index_col='building_id')<jupyter_output><empty_output><jupyter_text>Ordenamiento<jupyter_code>id = '1kt2VFhgpfRS72wtBOBy1KDat9LanfMZU'
downloaded = drive.CreateFile({'id': id})
downloaded.GetContentFile('test_values.csv')

test_values = pd.read_csv('test_values.csv', encoding='latin-1', dtype = {'building_id': 'int64', 'geo_level_2_id': 'int64', 'geo_level_3_id': 'int64',\
                                                                           'count_floors_pre_eq': 'int64', 'age': 'int64', 'area_percentage': 'int64', \
                                                                           'height_percentage': 'int64', 'land_surface_condition': 'category',\
                                                                           'foundation_type': 'category', 'roof_type': 'category', 'ground_floor_type': 'category',\
                                                                           'other_floor_type': 'category', 'position': 'category', 'plan_configuration': 'category',\
                                                                           'has_superstructure_adobe_mud': 'boolean', 'has_superstructure_mud_mortar_stone': 'boolean', \
                                                                           'has_superstructure_stone_flag': 'boolean', 'has_superstructure_cement_mortar_stone': 'boolean',\
                                                                           'has_superstructure_mud_mortar_brick': 'boolean', 'has_superstructure_cement_mortar_brick': 'boolean',\
                                                                           'has_superstructure_timber': 'boolean', 'has_superstructure_bamboo': 'boolean',\
                                                                           'has_superstructure_rc_non_engineered': 'boolean', 'has_superstructure_rc_engineered': 'boolean',\
                                                                           'has_superstructure_other': 'boolean', 'legal_ownership_status': 'category', 'count_families': 'int64', \
                                                                           'has_secondary_use': 'boolean', 'has_secondary_use_agriculture': 'boolean', 'has_secondary_use_hotel': 'boolean', \
                                                                           'has_secondary_use_rental': 'boolean', 'has_secondary_use_institution': 'boolean', 'has_secondary_use_school': 'boolean',\
                                                                           'has_secondary_use_industry': 'boolean', 'has_secondary_use_health_post': 'boolean', \
                                                                           'has_secondary_use_gov_office': 'boolean', 'has_secondary_use_use_police': 'boolean', 'has_secondary_use_other': 'boolean'})

test_values2 = test_values2.reset_index()
test_values = test_values['building_id']
test_values = test_values.to_frame()
test_values2 = test_values.merge(test_values2, on = 'building_id').set_index('building_id')
df = train_values2.merge(train_labels, left_index = True, right_index= True)
train_values2 = df.drop(columns='damage_grade')
train_labels = pd.DataFrame(df['damage_grade'])
test_values = pd.read_csv('test_values.csv', encoding='latin-1', dtype = {'building_id': 'int64', 'geo_level_2_id': 'int64', 'geo_level_3_id': 'int64',\
                                                                           'count_floors_pre_eq': 'int64', 'age': 'int64', 'area_percentage': 'int64', \
                                                                           'height_percentage': 'int64', 'land_surface_condition': 'category',\
                                                                           'foundation_type': 'category', 'roof_type': 'category', 'ground_floor_type': 'category',\
                                                                           'other_floor_type': 'category', 'position': 'category', 'plan_configuration': 'category',\
                                                                           'has_superstructure_adobe_mud': 'boolean', 'has_superstructure_mud_mortar_stone': 'boolean', \
                                                                           'has_superstructure_stone_flag': 'boolean', 'has_superstructure_cement_mortar_stone': 'boolean',\
                                                                           'has_superstructure_mud_mortar_brick': 'boolean', 'has_superstructure_cement_mortar_brick': 'boolean',\
                                                                           'has_superstructure_timber': 'boolean', 'has_superstructure_bamboo': 'boolean',\
                                                                           'has_superstructure_rc_non_engineered': 'boolean', 'has_superstructure_rc_engineered': 'boolean',\
                                                                           'has_superstructure_other': 'boolean', 'legal_ownership_status': 'category', 'count_families': 'int64', \
                                                                           'has_secondary_use': 'boolean', 'has_secondary_use_agriculture': 'boolean', 'has_secondary_use_hotel': 'boolean', \
                                                                           'has_secondary_use_rental': 'boolean', 'has_secondary_use_institution': 'boolean', 'has_secondary_use_school': 'boolean',\
                                                                           'has_secondary_use_industry': 'boolean', 'has_secondary_use_health_post': 'boolean', \
                                                                           'has_secondary_use_gov_office': 'boolean', 'has_secondary_use_use_police': 'boolean', 'has_secondary_use_other': 'boolean'})

test_values1 = test_values1.reset_index()
test_values = test_values['building_id']
test_values = test_values.to_frame()
test_values1 = test_values.merge(test_values1, on = 'building_id').set_index('building_id')
df = train_values1.merge(train_labels, left_index = True, right_index= True)
train_values1 = df.drop(columns='damage_grade')
train_labels = pd.DataFrame(df['damage_grade'])<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>train_values = train_values2.copy()
test_values = test_values2.copy()<jupyter_output><empty_output><jupyter_text>Continua la regresion<jupyter_code>index = df_index.index
cols = train_values.index.tolist()
not_in_index = []
for col in cols: 
  if col not in index: not_in_index.append(col)
valid_values = train_values.loc[index]
train_values = train_values.loc[not_in_index]

valid_labels = train_labels.loc[index]
train_labels = train_labels.loc[not_in_index]
idx1 = train_values.shape[0]
idx2 = valid_values.shape[0]
data_df = pd.concat([train_values, valid_values, test_values], sort=False)
cat_features = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'land_surface_condition',
                'foundation_type', 'roof_type', 'ground_floor_type', 'other_floor_type',
                'position', 'plan_configuration', 'legal_ownership_status',
                'count_families', 'range_age']
                    
num_features = ['count_floors_pre_eq', 'max_mean_area_percentage_id_1', 'mean_area_geo_level_1_id']
data_cat = pd.DataFrame(index = data_df.index, 
                  data = data_df,
                  columns = cat_features)
data_num = data_df.drop(columns = cat_features)
num_features = data_num.columns
data_num.shape
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import MinMaxScaler

enc = OneHotEncoder(drop='first')
enc.fit(data_cat)
data_cat_encoded = enc.transform(data_cat)
type(data_cat_encoded)
scaler = MinMaxScaler()
scaler.fit(data_num[:idx2])
data_num_scaled = scaler.transform(data_num)
type(data_num)
data_cat_encoded
from scipy.sparse import coo_matrix, hstack
data = hstack((data_cat_encoded,data_num_scaled))
data_cat_encoded
data = data.astype(dtype='float16')
X_train = data.tocsr()[:idx1]
X_valid = data.tocsr()[idx1:idx2+idx1]
X_test = data.tocsr()[idx2+idx1:]
y_train = train_labels['damage_grade'].values
y_valid = valid_labels['damage_grade'].values
train_values = train_values.astype(dtype= {'geo_level_1_id': 'category', 'geo_level_2_id': 'category',
                                           'geo_level_3_id': 'category', 'count_families':'category', 
                                           'count_materials': 'category'})

test_values = test_values.astype(dtype= {'geo_level_1_id': 'category', 'geo_level_2_id': 'category',
                                           'geo_level_3_id': 'category', 'count_families':'category', 
                                           'count_materials': 'category'})
#idx = train_values.shape[0]
#data_df = pd.concat([train_values, test_values], sort=False)

#data_cat = pd.DataFrame(index = data_df.index, 
#                  data = data_df, 
#                  columns = cat_features)

#data_num = data_df.drop(columns = cat_features)
#data = data.astype(dtype='float16')
#X_train = data.tocsr()[:idx]
#X_test = data.tocsr()[idx:]
# para el undersample poner train_labels2
#y_train = train_labels['damage_grade']
#from sklearn.model_selection import train_test_split
#X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.3, random_state=9)<jupyter_output><empty_output><jupyter_text>Regresion<jupyter_code>from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression(C = 0.9, verbose = 100, solver ='liblinear', max_iter=500, n_jobs=-1, random_state = 5)
log_reg.fit(X_train, y_train)
from sklearn.metrics import f1_score
# TEST SCORE
y_pred = log_reg.predict(X_valid)
f1_score(y_valid, y_pred, average='micro')
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, classification_report

pred = y_pred
score = f1_score(y_valid, pred, average='micro')
#score = accuracy_score(y_valid_split, pred)

cm = confusion_matrix(y_valid, pred)
report = classification_report(y_valid, pred)

print("f1_micro: ", score, "\n\n")
print(cm, "\n\n")
print(report, "\n\n")
# TRAIN SCORE
y_pred = log_reg.predict(X_train)
f1_score(y_train, y_pred, average='micro')<jupyter_output><empty_output><jupyter_text>Algunas cosas que se pueden probar; Tarda un rato largo el smote  Si quiero hacer un SMOTE<jupyter_code>from imblearn.over_sampling import SMOTE

sm = SMOTE(random_state=27)
X_train, y_train = sm.fit_sample(X_train, y_train)<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).
  "(https://pypi.org/project/six/).", FutureWarning)
/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.
  warnings.warn(message, FutureWarning)
/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>Si quiero OverSampling<jupyter_code>pip install -U imbalanced-learn
from imblearn.over_sampling import RandomOverSampler
sampling_strategy = "not majority"
ros = RandomOverSampler(sampling_strategy=sampling_strategy)
X_train, y_train = ros.fit_resample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>SI quiero un undersampling<jupyter_code>from imblearn.under_sampling import RandomUnderSampler 
sampling_strategy = "not minority"

rus = RandomUnderSampler(sampling_strategy=sampling_strategy)
X_train, y_train = rus.fit_resample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Ridge<jupyter_code>from sklearn.linear_model import RidgeClassifier
#penalty = 'l1'
ridge = RidgeClassifier(alpha= 10, max_iter=500, random_state = 5)
ridge.fit(X_train, y_train)
from sklearn.metrics import f1_score
# TEST SCORE
y_pred = ridge.predict(X_valid)
f1_score(y_valid, y_pred, average='micro')
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, classification_report

pred = y_pred
score = f1_score(y_valid, pred, average='micro')
#score = accuracy_score(y_valid_split, pred)

cm = confusion_matrix(y_valid, pred)
report = classification_report(y_valid, pred)

print("f1_micro: ", score, "\n\n")
print(cm, "\n\n")
print(report, "\n\n")
# TRAIN SCORE
y_pred = ridge.predict(X_train)
f1_score(y_train, y_pred, average='micro')
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, classification_report

pred = y_pred
score = f1_score(y_train, pred, average='micro')
#score = accuracy_score(y_valid_split, pred)

cm = confusion_matrix(y_train, pred)
report = classification_report(y_train, pred)

print("f1_micro: ", score, "\n\n")
print(cm, "\n\n")
print(report, "\n\n")
X_test.shape<jupyter_output><empty_output><jupyter_text>Linear Discriminant<jupyter_code>from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

lda = LinearDiscriminantAnalysis()
lda.fit(X_train.toarray(), y_train)
from sklearn.metrics import f1_score
# TEST SCORE
y_pred = lda.predict(X_valid.to_array())
f1_score(y_valid, y_pred, average='micro')
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, classification_report

pred = y_pred
score = f1_score(y_valid, pred, average='micro')
#score = accuracy_score(y_valid_split, pred)

cm = confusion_matrix(y_valid, pred)
report = classification_report(y_valid, pred)

print("f1_micro: ", score, "\n\n")
print(cm, "\n\n")
print(report, "\n\n")
# TRAIN SCORE
y_pred = ridge.predict(X_train)
f1_score(y_train, y_pred, average='micro')
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, classification_report

pred = y_pred
score = f1_score(y_train, pred, average='micro')
#score = accuracy_score(y_valid_split, pred)

cm = confusion_matrix(y_train, pred)
report = classification_report(y_train, pred)

print("f1_micro: ", score, "\n\n")
print(cm, "\n\n")
print(report, "\n\n")<jupyter_output>f1_micro:  0.7799196448640144 


[[109313   7685   1642]
 [ 19091  77204  22345]
 [  4378  23190  91072]] 


              precision    recall  f1-score   support

           1       0.82      0.92      0.87    118640
           2       0.71      0.65      0.68    118640
           3       0.79      0.77      0.78    118640

    accuracy                           0.78    355920
   macro avg       0.78      0.78      0.78    355920
weighted avg       0.78      0.78      0.78    355920<jupyter_text>Feature Selection<jupyter_code>import joblib
from mlxtend.feature_selection import SequentialFeatureSelector as sfs
log_reg = LogisticRegression(C = 1.0, n_jobs=-1, random_state = 5)

sfs1 = sfs(log_reg,
           k_features=100,
           forward=True,
           floating=False,
           verbose=2,
           scoring='f1_micro',
           cv=3)

# Perform SFFS
sfs1 = sfs1.fit(X_train, y_train)
feat_cols = list(sfs1.k_feature_idx_)
print(feat_cols)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>probas = pd.DataFrame(log_reg.predict_proba(X_valid), index = valid_labels['damage_grade'].index, columns=['proba_1', 'proba_2', 'proba_3'])
y_predicted = log_reg.predict(X_valid)
probas['predicted_RL_7432'] = y_predicted
probas
probas.to_csv('3_7432_RL_valid_probas.csv')
predicted_df = pd.DataFrame(y_pred.astype(np.int64), index = y_valid.index, columns=['damage_grade'])
predicted_df.to_csv('mas_cortito.csv')
from google.colab import files
files.download("mas_cortito.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11239848402815376, 'fraction_numerical': 0.022035733622089874, 'mean_word_length': 3.3656346017489955, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 110, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '270945', 'license': 'MIT', 'n_tokens_mistral': 6015, 'n_tokens_neox': 5735, 'n_words': 991, 'path': 'data/3 - RL-TodosLosFeatures.ipynb', 'repo_name': 'gsabatino9/OrganizacionDeDatos', 'size': '18470'}