<jupyter_start><jupyter_text>AceleraDev DataScience  Setuphttps://www.kaggle.com/c/house-prices-advanced-regression-techniques/data<jupyter_code>#lendo os pacotes
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
df = pd.read_csv('train.csv')<jupyter_output><empty_output><jupyter_text>Analysis  Selecao por completude<jupyter_code>#Criando um dataframe auxliar para analisar a consistencia das variaveis
cons = pd.DataFrame({'colunas' : df.columns,
                    'tipo': df.dtypes,
                    'missing' : df.isna().sum(),
                    'size' : df.shape[0],
                    'unicos': df.nunique()})
cons['percentual'] = round(cons['missing'] / cons['size'],2)
cons.percentual.plot.hist( bins = 5)<jupyter_output><empty_output><jupyter_text>Removendo colunas com dados missing<jupyter_code>print('Contagem de colunas com ATÉ 20% de dados faltantes', cons[cons.percentual < 0.2].shape[0])
print('Contagem de colunas com 0% de dados faltantes',  cons[cons.percentual == 0].shape[0])
cons[cons.percentual == 0]['tipo'].value_counts()
cons['completa'] = ['completa' if x == 0 else 'faltante' for x in cons['percentual']]
mantem = list(cons[cons['completa'] == 'completa']['colunas'])
df = df[mantem]
colunas_numericas = list(cons[((cons['tipo'] != 'object') &
                              (cons['completa'] == 'completa'))]['colunas'])<jupyter_output><empty_output><jupyter_text>Exploração<jupyter_code># Analise univariavel
for coluna in colunas_numericas:
    print(coluna)
    df[coluna].plot.hist(bins = 50, log= True)
    plt.show()
#Analisando a correlacao entre as variaveis númericas
plt.figure(figsize = (20,20))
sns.heatmap(df[colunas_numericas].corr().round(2), annot= True)
correlacionadas = ['GarageArea', 'GarageCars', 'GrLivArea', 'OverallQual']<jupyter_output><empty_output><jupyter_text>Analisando as features com yellowbrick<jupyter_code>pip install yellowbrick
#Removendo a coluna ID
colunas_numericas.remove('Id')
df = df[colunas_numericas]
y_train = df['SalePrice']
X_train = df.drop(columns = 'SalePrice')
from yellowbrick.features import Rank1D

visualizer = Rank1D(algorithm='shapiro')

visualizer.fit(X_train, y_train)           
visualizer.transform(X_train)        
visualizer.show()              
from yellowbrick.features import PCA


visualizer = PCA(scale=True, proj_features=True, projection=2)
visualizer.fit_transform(X_train[correlacionadas], y_train)
visualizer.show()
from yellowbrick.target import FeatureCorrelation

features = list(X_train.columns)

visualizer = FeatureCorrelation(labels=features)

visualizer.fit(X_train, y_train)        
visualizer.show()<jupyter_output>/Users/tuliosouza/opt/anaconda3/envs/aceleradev_ds/lib/python3.8/site-packages/sklearn/base.py:193: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.
  warnings.warn('From version 0.24, get_params will raise an '<jupyter_text>Treinando o modelo<jupyter_code>pip install sklearn
from sklearn.linear_model import LinearRegression
reg= LinearRegression()
reg.fit(X_train, y_train)
colunas_treinamento = X_train.columns
X_test = pd.read_csv('test.csv')
y_test = pd.read_csv('sample_submission.csv')
y_test = y_test['SalePrice']
X_test= X_test[colunas_treinamento].fillna(df[colunas_treinamento].mean())
y_pred = reg.predict(X_test)
from sklearn.metrics import mean_squared_error
erro_normal = mean_squared_error(y_pred=y_pred, y_true=y_test)
erro_normal<jupyter_output><empty_output><jupyter_text>Aplicando o Feature Selection<jupyter_code>from sklearn.feature_selection import RFE
rfe = RFE(reg)
rfe.fit(X_train, y_train)
pd.DataFrame({'coluna':X_train.columns,
              'bool': rfe.get_support(),
              'coeficientes': pd.Series(reg.coef_)})
X_train_importante = rfe.transform(X_train)
X_test_importante = rfe.transform(X_test)
reg.fit(X_train_importante, y_train)
y_pred_imp = reg.predict(X_test_importante)
erro_imp = mean_squared_error(y_pred=y_pred_imp, y_true=y_test)
erro_imp
from yellowbrick.model_selection import RFECV


# Instantiate RFECV visualizer with a linear SVM classifier
visualizer = RFECV(reg)

visualizer.fit(X_train, y_train)        # Fit the data to the visualizer
visualizer.show()           # Finalize and render the figure<jupyter_output>/Users/tuliosouza/opt/anaconda3/envs/aceleradev_ds/lib/python3.8/site-packages/sklearn/base.py:193: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.
  warnings.warn('From version 0.24, get_params will raise an '
/Users/tuliosouza/opt/anaconda3/envs/aceleradev_ds/lib/python3.8/site-packages/sklearn/base.py:193: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.
  warnings.warn('From version 0.24, get_params will raise an '<jupyter_text>Aplicando PCA<jupyter_code>from sklearn.decomposition import PCA
pca = PCA(0.95)
pca.fit(X_train)
pca.explained_variance_ratio_
X_train_pca = pca.transform(X_train)
X_test_pca = pca.transform(X_test)
reg = LinearRegression()
reg.fit(X_train_pca, y_train)
y_pred_pca = reg.predict(X_test_pca)
erro_pca = mean_squared_error(y_pred=y_pred_pca, y_true=y_test)
pd.DataFrame({'erro' : [erro_normal, erro_imp, erro_pca]}).plot(kind = 'bar', log = True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1144512857660268, 'fraction_numerical': 0.011227816008692503, 'mean_word_length': 5.768382352941177, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '63946', 'license': 'MIT', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1958, 'n_words': 392, 'path': 'module_6/Semana 6 - Aceleradev.ipynb', 'repo_name': 'JedersonLuz/Codenation_AceleraDev_DataScience', 'size': '5522'}