<jupyter_start><jupyter_text>House Prices - Advanced Regression Techniques<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost
df_train = pd.read_csv("data/train.csv")
df_test = pd.read_csv("data/test.csv")
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    print(df_train.head())
df_train.shape
df_test.shape
print("Percentage of null values for df_train")

with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    percentage_null_values = df_train.isnull().sum()*100/df_train.shape[0]
    
    for col,val in percentage_null_values.iteritems():
        if val!=0:
            print(col,"    -    ",val)
print("Percentage of null values for df_test")

with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    percentage_null_values = df_test.isnull().sum()*100/df_test.shape[0]
    
    for col,val in percentage_null_values.iteritems():
        if val!=0:
            print(col,"    -    ",val)
            
df = pd.concat([df_train,df_test])
df.columns
threshold = 20
col_with_null_beyond_threshold = []
col_with_null_below_threshold = []

print("Percentage of null values for total df")

with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    percentage_null_values = df.isnull().sum()*100/df.shape[0]
    
    for col,val in percentage_null_values.iteritems():
        if col=='SalePrice':
            continue
        if val!=0:
            print(col,"    -    ",val)
            if val>threshold:
                col_with_null_beyond_threshold.append(col)
            else:
                col_with_null_below_threshold.append(col)
df.shape
col_with_null_beyond_threshold
df = df.drop(col_with_null_beyond_threshold,axis=1)
df.shape
df.iloc[df_train.shape[0]-1,:]['SalePrice']
col_with_null_below_threshold
for col in col_with_null_below_threshold:
    df[col] = df[col].fillna(df[col].mode()[0])
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    print(df.isna().sum())
df.shape
categorical_columns = []
for col in df.columns:
    if df[col].dtype == object:
        categorical_columns.append(col)
for index,col in enumerate(categorical_columns):
    
    df1 = pd.get_dummies(df[col],drop_first = True)
    df.drop([col],axis=1,inplace=True)
    df = pd.concat([df,df1],axis=1)

        
df.iloc[df_train.shape[0]-1,:]['SalePrice']
df = df.loc[:,~df.columns.duplicated()]

df.shape
df.iloc[df_train.shape[0]-1,:]['SalePrice']
df.drop(['Id'],axis=1,inplace=True)
df.iloc[df_train.shape[0]-1,:]['SalePrice']
'SalePrice' in df.columns
dfTrain = df.iloc[:df_train.shape[0],:]
dfTest = df.iloc[df_train.shape[0]:,:]
dfTrain['SalePrice']
dfTest.drop(['SalePrice'],axis=1,inplace=True)
dfTrain['SalePrice']
dfTest.shape
y = dfTrain['SalePrice']
x = dfTrain.drop(['SalePrice'],axis=1)
y
import xgboost
regressor=xgboost.XGBRegressor()
'SalePrice' in x.columns
bst = regressor.fit(x,y)

'SalePrice' in dfTest.columns
y_pred = regressor.predict(dfTest)
from sklearn.metrics import mean_squared_error
print()
pred=pd.DataFrame(y_pred)
sub_df=pd.read_csv('data/sample_submission.csv')
datasets=pd.concat([sub_df['Id'],pred],axis=1)
datasets.columns=['Id','SalePrice']
datasets.to_csv('sample_submission.csv',index=False)
# Alley : NA - No alley access
# BsmtQual : NA - No basement
# BsmtCond : NA - No basement
# BsmtExposure : NA - No basement
# BsmtFinType1 : NA - No basemen
# BsmtFinType2 : NA - No basemen
# FireplaceQu : NA - No fireplace
# GarageType : NA  - No Garage
# GarageFinish : NA  - No Garage
# GarageFinish : NA  - No Garage
# GarageCond : NA  - No Garage
# PoolQC : NA - No pool
# Fence : NA - No Fence
# MiscFeature : NA - No Other luxury
df.describe()
categorical_col = []
continous_col = []
for col in df.columns:
    if df[col].dtype == object:
        categorical_col.append(col)
    else:
        continous_col.append(col)
#drop id since it is unique for each data point
col_with_null_beyond_threshold<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1334272962179939, 'fraction_numerical': 0.009161381254404511, 'mean_word_length': 4.26980198019802, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '295090', 'license': 'MIT', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1506, 'n_words': 310, 'path': 'House-Price-Predict.ipynb', 'repo_name': 'kungwanin/House-Price-prediction', 'size': '4257'}