<jupyter_start><jupyter_text>Load Dattasets from Kaggle API<jupyter_code># Author:  <NAME> <<EMAIL>>
# License: BSD 3 clause

import os
import shutil
# Create kaggle folder in Colab
kaggle_dir = "/kaggle/input"
directory = "house-prices-advanced-regression-techniques"
path = os.path.join(kaggle_dir, directory)
try: 
    os.makedirs(kaggle_dir, exist_ok=True)
    print("Directory '%s' created " %kaggle_dir)
except OSError as error:
    print("Directory '%s' can not be created")

# Access to Kaggle Database
os.environ['KAGGLE_USERNAME'] = "phanttan" 
os.environ['KAGGLE_KEY'] = "<KEY>" 
!kaggle competitions download -c house-prices-advanced-regression-techniques -p house-prices-advanced-regression-techniques

# Move dataset to new folder 
shutil.move(directory, kaggle_dir)<jupyter_output>Directory '/kaggle/input' created 
Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)
Downloading data_description.txt to house-prices-advanced-regression-techniques
  0% 0.00/13.1k [00:00<?, ?B/s]
100% 13.1k/13.1k [00:00<00:00, 12.6MB/s]
Downloading sample_submission.csv to house-prices-advanced-regression-techniques
  0% 0.00/31.2k [00:00<?, ?B/s]
100% 31.2k/31.2k [00:00<00:00, 32.0MB/s]
Downloading train.csv to house-prices-advanced-regression-techniques
  0% 0.00/450k [00:00<?, ?B/s]
100% 450k/450k [00:00<00:00, 81.7MB/s]
Downloading test.csv to house-prices-advanced-regression-techniques
  0% 0.00/441k [00:00<?, ?B/s]
100% 441k/441k [00:00<00:00, 62.3MB/s]<jupyter_text>Import packages and load dataset<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns

from sklearn.model_selection import train_test_split, RandomizedSearchCV, KFold, cross_val_score
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer

from sklearn.metrics import mean_squared_error

import xgboost as xgb

import warnings
warnings.filterwarnings('ignore')
train_full = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
test_full = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
train_full.shape, test_full.shape<jupyter_output><empty_output><jupyter_text>Data Preprocessing1.   Check duplications in Data2.   Transformation pipelines<jupyter_code># Check for duplicates for train data
idsUnique = len(set(train_full.Id))
idsTotal = train_full.shape[0]
idsDupli = idsTotal - idsUnique

if idsDupli != 0:
    print("Have duplicated Id in train")
else:
    print("No dupicated Id in train")

# Check for duplicates for test data
idsUnique = len(set(test_full.Id))
idsTotal = test_full.shape[0]
idsDupli = idsTotal - idsUnique

if idsDupli != 0:
    print("Have duplicated Id in test")
else:
    print("No dupicated Id in test")
train = train_full.drop('Id', axis=1)
test = test_full.drop('Id', axis=1)
X = train.drop('SalePrice', axis=1)
y = train.SalePrice
# List out all features that are categorical
numerical_cols = X.select_dtypes(include= ['int64', 'float64']).columns
categorical_cols = X.select_dtypes(include = ["object"]).columns
# Build Transformer pipeline
numerical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='mean')),
                                         ('scaler', StandardScaler(with_mean=False))
                                         ])
categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')),
                                          ('onehot', OneHotEncoder(handle_unknown='ignore')),
                                           ('scaler', StandardScaler(with_mean=False))
                                           ])
preprocessor = ColumnTransformer(transformers=[('num', numerical_transformer, numerical_cols),
                                              ('cat', categorical_transformer, categorical_cols)])<jupyter_output><empty_output><jupyter_text>Note: 1.   We chose OHE(OneHot Encoder), not Label Encoder because Label Encoder is not fair with all Categories Variables. 2.   A disadvantage of OHE is high cardinality, we can use PCA to reduce data dimension.<jupyter_code># Split data for Training and Validation
X_train, X_val, y_train, y_val = train_test_split(X, y)<jupyter_output><empty_output><jupyter_text>Define the Model<jupyter_code>xgb_reg = xgb.XGBRegressor(random_state=42, seed=123, silent=True)
ames_pipe = Pipeline(steps=[('preprocessor_transformers', preprocessor),
                            ('model', xgb_reg)])

ames_pipe<jupyter_output><empty_output><jupyter_text>Fine-Tuning Model : Using RandomizedSearchCV<jupyter_code>xgb_param_grid = {'model__subsample': np.arange(0.05, 1, 0.05),
                  'model__max_depth': np.arange(3,20,1),
                  'model__colsample_bytree': np.arange(0.1, 1.05, 0.05)}
randomized_neg_mse = RandomizedSearchCV(estimator= ames_pipe,
                                        param_distributions=xgb_param_grid,
                                        scoring='neg_mean_squared_error',
                                        cv=4)
randomized_neg_mse.fit(X_train, y_train)
# best_rmse = np.sqrt(np.abs(randomized_neg_mse.best_score_))
# best_param = randomized_neg_mse.best_params_
best_estimator= randomized_neg_mse.best_estimator_
y_pred = best_estimator.predict(X_val)
xgb_score = np.sqrt(mean_squared_error(y_val, y_pred))
def rmse_cv(model, X, y, n_folds):
    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(X)
    rmse= np.sqrt(-cross_val_score(model, X, np.log(y), scoring="neg_mean_squared_error", cv = kf))
    return rmse.mean()
rmse = rmse_cv(best_estimator, X_val, y_val, 5)
rmse
y_pred = best_estimator.predict(test)<jupyter_output><empty_output><jupyter_text>Create Submission File<jupyter_code>sample_sub = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
sample_sub.Id.shape, y_pred.shape
final_data = {'Id': sample_sub.Id, 'SalePrice': y_pred}
final_submission = pd.DataFrame(data=final_data).to_csv('submission_file.csv', 
                                                        index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11297271421732887, 'fraction_numerical': 0.027924046593266316, 'mean_word_length': 4.112561174551387, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '289680', 'license': 'MIT', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2093, 'n_words': 429, 'path': 'Pipeline_RandomizedSearchCV_XGBoostRegression_on_Ames_Dataset.ipynb', 'repo_name': 'Phanttan/Kaggle_house-prices-advanced-regression-techniques', 'size': '6307'}