<jupyter_start><jupyter_text>House Prices: Advanced Regression Techniques : Kaggle Competition  Import Libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Import Data<jupyter_code>df=pd.read_csv('data_cleaned.csv')
df.head()
df.shape<jupyter_output><empty_output><jupyter_text>Step1: Check for missing values<jupyter_code>fig, ax = plt.subplots(figsize=(20,5))         # To change fig shape for better representation
sns.heatmap(df.isnull(),yticklabels=False,cbar=False, ax=ax)
def missing_zero_values_table(dataframe):
        zero_val = (dataframe == 0.00).astype(int).sum(axis=0)
        mis_val = dataframe.isnull().sum()
        mis_val_percent = 100 * dataframe.isnull().sum() / len(dataframe)
        mz_table = pd.concat([zero_val, mis_val, mis_val_percent], axis=1)
        mz_table = mz_table.rename(
        columns = {0 : 'Zero Values', 1 : 'Missing Values', 2 : '% of Total Values'})
        zero_val = (dataframe == 0.00).astype(int).sum(axis=0)
        mis_val = dataframe.isnull().sum()
        mis_val_percent = 100 * dataframe.isnull().sum() / len(dataframe)
        mz_table = pd.concat([zero_val, mis_val, mis_val_percent], axis=1)
        mz_table = mz_table.rename(
        columns = {0 : 'Zero Values', 1 : 'Missing Values', 2 : '% of Total Values'})
        mz_table['Data Type'] = dataframe.dtypes
        mz_table = mz_table[
            mz_table.iloc[:,1] != 0].sort_values(
        '% of Total Values', ascending=False).round(1)
        print ("Your selected dataframe has " + str(dataframe.shape[1]) + " columns and " + str(dataframe.shape[0]) + " Rows.\n"      
            "There are " + str(mz_table.shape[0]) +
              " columns that have missing values.")
        return mz_table
missing_zero_values_table(df)
df.info()
import pandas as pd
import numpy as np
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
X = df.iloc[:,1:76]  #independent columns
y = df.iloc[:,-1]    #target column i.e price range
X.head()
y.head()
##HAndle Categorical Features
nums = list(X.select_dtypes(include=['int64','float64']).columns)
nums
Num_data=X[nums]
Num_data.shape
Num_data.head()
y.head()
#apply SelectKBest class to extract top 10 best features
bestfeatures = SelectKBest(score_func=chi2, k=10)
fit = bestfeatures.fit(Num_data,y)
dfscores = pd.DataFrame(fit.scores_)
dfcolumns = pd.DataFrame(Num_data.columns)

#concat two dataframes for better visualization 
featureScores = pd.concat([dfcolumns,dfscores],axis=1)
featureScores.columns = ['Specs','Score']  #naming the dataframe columns
featureScores
print(featureScores.nlargest(10,'Score'))  #print 10 best features
from sklearn.ensemble import ExtraTreesClassifier
import matplotlib.pyplot as plt
model = ExtraTreesClassifier()
model.fit(Num_data,y)
print(model.feature_importances_) #use inbuilt class feature_importances of tree based classifiers

#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=Num_data.columns)
feat_importances.nlargest(20).plot(kind='barh')
plt.show()
num_tar_data=pd.concat([Num_data,y],axis=1)
num_tar_data.head()
import seaborn as sns
#get correlations of each features in dataset
corrmat = num_tar_data.corr()
top_corr_features = corrmat.index
plt.figure(figsize=(30,30))
#plot heat map
g=sns.heatmap(num_tar_data[top_corr_features].corr(),annot=True,cmap="RdYlGn")
corrmat
corrmat.shape[0]
features_SalePrice=corrmat.iloc[-1:,:]
features_SalePrice
(features_SalePrice.iloc[:,5])[0]
imp_num_feature=[]
for i in range(0,corrmat.shape[0]-1):
    if(features_SalePrice.iloc[:,i][0]>0.2): #if co-relation value>0.2 its imp feature
        imp_num_feature.append(nums[i])
imp_num_feature
non_num_feature=[]
for i in range(0,corrmat.shape[0]-1):
    if(features_SalePrice.iloc[:,i][0]<0.2): #if co-relation value>0.2 its imp feature
        non_num_feature.append(nums[i])
non_num_feature<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12768584835761917, 'fraction_numerical': 0.019264015806371944, 'mean_word_length': 5.0357675111773474, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '586981', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1424, 'n_words': 296, 'path': 'Extracting Imp Numerical Feature.ipynb', 'repo_name': 'malavika8/HousePricePrediction', 'size': '4049'}