<jupyter_start><jupyter_text>Checking column types<jupyter_code>numerical_columns = train_data.select_dtypes(exclude='object')
numerical_columns
numerical_columns.shape<jupyter_output><empty_output><jupyter_text>We have 14 numerical columns. 'cont0', 'cont1', 'cont2', ..., 'cont13'<jupyter_code>categorical_columns = train_data.select_dtypes(include='object')
categorical_columns
categorical_columns.shape<jupyter_output><empty_output><jupyter_text>We have 10 categorical columns. 'cat0', 'cat1', 'cat2', ..., 'cat9'  Checking null/NaN values  Train data<jupyter_code>train_data.isnull().sum()<jupyter_output><empty_output><jupyter_text>Test data<jupyter_code>test_data.isnull().sum()<jupyter_output><empty_output><jupyter_text>Dropping the ID column<jupyter_code>train_data = train_data.drop(columns=['id'])<jupyter_output><empty_output><jupyter_text>Checking outliers<jupyter_code>numerical_columns_target = numerical_columns['target'].copy()
numerical_columns.drop(columns=['id', 'target'], inplace=True)
figure = plt.figure(figsize=(10, 15))
train_data_numerical = train_data[numerical_columns.columns]

for i in range(len(numerical_columns.columns)):
    figure.add_subplot(8, 2, i+1)
    sns.boxplot(train_data_numerical.iloc[:, i])

plt.tight_layout()
plt.show()
figure = plt.figure(figsize=(10, 15))
train_data_numerical = train_data[numerical_columns.columns]

for i in range(len(numerical_columns.columns)):
    figure.add_subplot(8, 2, i+1)
    sns.scatterplot(x=train_data_numerical.iloc[:, i], y=numerical_columns_target)

plt.tight_layout()
plt.show()
#removing outliers (when target=0)
train_data = train_data[train_data['target'] > 0]
train_data.shape<jupyter_output><empty_output><jupyter_text>Checking the categorical columns<jupyter_code>figure = plt.figure(figsize=(10, 15))
train_data_categorical = train_data[categorical_columns.columns]

for i in range(len(categorical_columns.columns)):
    figure.add_subplot(8, 2, i+1)
    sns.countplot(x=train_data_categorical.iloc[:, i])

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Transforming categorical columns to numerical<jupyter_code>from sklearn.preprocessing import LabelEncoder

for i in range(len(categorical_columns.columns)):
    label_encoder = LabelEncoder()
    train_data_categorical[train_data_categorical.columns[i]] = label_encoder.fit_transform(train_data_categorical.iloc[:, i])
train_data_categorical.head()
#dropping original columns
train_data.drop(columns=train_data_categorical.columns, inplace=True)
train_data = pd.concat([train_data, train_data_categorical], axis=1)
train_data.head()
#checking columns types
train_data.dtypes<jupyter_output><empty_output><jupyter_text>Checking distplot<jupyter_code>figure = plt.figure(figsize=(10, 20))

for i in range(len(train_data.columns)):
    figure.add_subplot(13, 2, i+1)
    ax = sns.distplot(x=train_data.iloc[:, i])
    ax.set(xlabel=train_data.columns[i], ylabel='Destiny')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Checking correlation<jupyter_code>plt.figure(figsize=(10, 6))
corr = train_data.corr()
sns.heatmap(corr[['target']], annot=True)<jupyter_output><empty_output><jupyter_text>Models<jupyter_code>from sklearn.model_selection import train_test_split

X = train_data.drop(columns=['target'])
y = train_data['target']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
from sklearn.ensemble import RandomForestRegressor
import lightgbm as lgb
import xgboost as xgb
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.linear_model import LinearRegression, Lasso, ElasticNet, SGDRegressor
from sklearn.svm import SVR

rf = RandomForestRegressor(random_state=19, n_jobs=-1)
lgb_model = lgb.LGBMRegressor(random_state=19, n_jobs=-1)
xgb_model = xgb.XGBRegressor(random_state=19, n_jobs=-1)
dt = DecisionTreeRegressor(random_state=19)
et = ExtraTreesRegressor(random_state=19, n_jobs=-1)
lr = LinearRegression()
l = Lasso(random_state=19)
en = ElasticNet(random_state=19)
sgd = SGDRegressor(random_state=19)
svr = SVR()
from sklearn.metrics import mean_squared_error

models = []
accuracies = []<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>rf.fit(X_train, y_train)
predicted = rf.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Random Forest')<jupyter_output><empty_output><jupyter_text>LGBM<jupyter_code>lgb_model.fit(X_train, y_train)
predicted = lgb_model.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('LGBM')<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>xgb_model.fit(X_train, y_train)
predicted = xgb_model.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('XGBoost')<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>dt.fit(X_train, y_train)
predicted = dt.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Decision Tree')<jupyter_output><empty_output><jupyter_text>Extra Trees<jupyter_code>et.fit(X_train, y_train)
predicted = et.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Extra Trees')<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code>lr.fit(X_train, y_train)
predicted = lr.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Linear Regression')<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>l.fit(X_train, y_train)
predicted = l.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Lasso')<jupyter_output><empty_output><jupyter_text>Elastic Net<jupyter_code>en.fit(X_train, y_train)
predicted = en.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('Elastic Net')<jupyter_output><empty_output><jupyter_text>SGD<jupyter_code>sgd.fit(X_train, y_train)
predicted = sgd.predict(X_test)
accuracies.append(np.sqrt(mean_squared_error(y_test, predicted)))
models.append('SGD')<jupyter_output><empty_output><jupyter_text>Pivot Table<jupyter_code>pt = pd.DataFrame({
    'Model': models,
    'Accuracy': accuracies
})

pivot_table = pd.pivot_table(pt, index='Model')
pivot_table<jupyter_output><empty_output><jupyter_text>Tunning LightGBM hyperparameters using Optuna<jupyter_code>import optuna
from sklearn.model_selection import KFold

def objective(trial):
    
    params = {
        'objective': 'regression',
        'metric': 'rmse',
        'verbosity': -1,
        'boosting_type': 'gbdt',
        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
        'num_leaves': trial.suggest_int('num_leaves', 2, 256),
        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
        'random_state': 19
    }
    
    fold = KFold(n_splits=7, shuffle=True, random_state=19)
    scores = []
    
    for train_index, test_index in fold.split(X):
        
        X_train = X.iloc[train_index]
        y_train = y.iloc[train_index]
        X_test = X.iloc[test_index]
        y_test = y.iloc[test_index]
    
        dtrain = lgb.Dataset(X_train, label=y_train)
    
        gbm = lgb.train(params, dtrain)
        predicted = gbm.predict(X_test)
        scores.append(np.sqrt(mean_squared_error(y_test, predicted)))
        
    return np.mean(scores) 
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=100)
study.best_trial.value
study.best_params
lgb_optuna = lgb.LGBMRegressor(**study.best_params)

lgb_optuna.fit(X_train, y_train)
predicted = lgb_optuna.predict(X_test)
np.sqrt(mean_squared_error(y_test, predicted))<jupyter_output>[LightGBM] [Warning] lambda_l1 is set=5.231399442791927e-08, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.231399442791927e-08
[LightGBM] [Warning] feature_fraction is set=0.43442171838042865, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.43442171838042865
[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1
[LightGBM] [Warning] lambda_l2 is set=9.641225535858149e-06, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.641225535858149e-06
[LightGBM] [Warning] bagging_fraction is set=0.8506937910720961, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8506937910720961<jupyter_text>Tunning XGBoost hyperparameters using Optuna<jupyter_code>import optuna

def objective(trial):
    
    param = {
        "verbosity": 0,
        "objective": "reg:linear",
        "metrics": "rmse",
        "booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]),
        "lambda": trial.suggest_float("lambda", 1e-8, 1.0, log=True),
        "alpha": trial.suggest_float("alpha", 1e-8, 1.0, log=True),
        "random_state": 19
    }

    if param["booster"] == "gbtree" or param["booster"] == "dart":
        param["max_depth"] = trial.suggest_int("max_depth", 1, 9)
        param["eta"] = trial.suggest_float("eta", 1e-8, 1.0, log=True)
        param["gamma"] = trial.suggest_float("gamma", 1e-8, 1.0, log=True)
        param["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"])
    if param["booster"] == "dart":
        param["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"])
        param["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"])
        param["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0, log=True)
        param["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True)
        
    fold = KFold(n_splits=7, shuffle=True, random_state=19)
    scores = []
    
    for train_index, test_index in fold.split(X):
        
        X_train = X.iloc[train_index]
        y_train = y.iloc[train_index]
        X_test = X.iloc[test_index]
        y_test = y.iloc[test_index]
        
        dtrain = xgb.DMatrix(X_train, label=y_train)
        dtest = xgb.DMatrix(X_test, label=y_test)

        xg = xgb.train(param, dtrain)
        predicted = xg.predict(dtest)
        scores.append(np.sqrt(mean_squared_error(y_test, predicted)))
    
    return np.mean(scores) 
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=100)
study.best_trial.value
study.best_params
xgb_optuna = xgb.XGBRegressor(**study.best_params)

xgb_optuna.fit(X_train, y_train)
predicted = xgb_optuna.predict(X_test)
np.sqrt(mean_squared_error(y_test, predicted))<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>test_data_categorical = test_data.select_dtypes(include='object')

for i in range(len(test_data_categorical.columns)):
    label_encoder = LabelEncoder()
    test_data_categorical[test_data_categorical.columns[i]] = label_encoder.fit_transform(test_data_categorical.iloc[:, i])
test_data.drop(columns=test_data_categorical.columns, inplace=True)
test_data = pd.concat([test_data, test_data_categorical], axis=1)
prediction = lgb_optuna.predict(test_data.drop(columns=['id']))

submission = pd.DataFrame({
    'id': test_data['id'],
    'target': prediction
})

submission.to_csv('submission_tabular.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13234661388864985, 'fraction_numerical': 0.02615953876602702, 'mean_word_length': 6.551656920077972, 'pattern_counts': {'":': 9, '<': 96, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '448306', 'license': 'MIT', 'n_tokens_mistral': 4565, 'n_tokens_neox': 4286, 'n_words': 605, 'path': 'Machine Learning/Tabular Playground Series/February 2021/tabular.ipynb', 'repo_name': 'rafaelgreca/AI-Projects', 'size': '11621'}