<jupyter_start><jupyter_text>Structure1. Data exploration2. Feature Engineering3. Train Test Split4. Model evaluation and metrics5. Ensembling6. Results.  Data Exploration<jupyter_code>dfs = DataFrameSummary(train_df)
dfs.columns_types
dfs.columns_stats
dfs["Body fat (%)"]
target = train_df["Body fat (%)"]
def create_ratio(df, col1, col2):
    col_name = f"{col1}_{col2}_ratio"
    df[col_name] = df[col1]/df[col2]
    return col_name

def feature_engineering_create_ratios(df):
    """Do not pass target in the df. It will be scaled."""
    columns = list(df.columns)
    ratio_list = combinations(columns, 2)
    ret = df.copy()
    for col1, col2 in ratio_list:
        create_ratio(ret, col1, col2)
    return ret<jupyter_output><empty_output><jupyter_text>Find outliers.We need to scatter plot all the columns to find outliers. Below cell does that.<jupyter_code>cols = list(train_df.columns)
n = math.ceil(math.sqrt(len(cols)))
columns=n
rows=n
fig, ax_array = plt.subplots(rows, columns, squeeze=False)
fig.set_size_inches(24, 24)
for i, ax_row in enumerate(ax_array):
    for j, axes in enumerate(ax_row):
        if rows * i + j < len(cols):
            col = cols[rows * i + j]
            axes.set_title(col)
            axes.scatter(list((range(201))), train_df[col])
plt.show()<jupyter_output><empty_output><jupyter_text>Remove outliersWe can easily eyeball outliers in the above graph. Now we can cleanit using statistics<jupyter_code>train_df = remove_outliers(train_df.drop(["Body fat (%)"], axis=1))
#train_df, target = remove_outliers(train_df.drop(["Body fat (%)"], axis=1), train_df["Body fat (%)"])
test_df["Height"] = test_df["Height (inches)"] * 2.54
test_df.drop(["Id", "Height (inches)"], axis=1, inplace=True)
train_processed = feature_engineering_create_ratios(train_df)
test_processed = feature_engineering_create_ratios(test_df)
from sklearn.metrics import mean_absolute_error
from lightgbm.sklearn import LGBMRegressor
from xgboost import XGBRegressor
from sklearn.ensemble import ExtraTreesRegressor, AdaBoostRegressor, BaggingRegressor,GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from sklearn.linear_model import LinearRegression, Ridge, Lasso
models = [ RandomForestRegressor, GradientBoostingRegressor, LGBMRegressor, XGBRegressor, AdaBoostRegressor,
          Ridge, Lasso]<jupyter_output><empty_output><jupyter_text>Hyperparameter optimization<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>params = {
    "n_estimators": [1, 20, 50, 100, 200, 500],
    "max_depth": [3, 10, 15, 20],
    "max_features": ["auto", "sqrt", "log2", 0.5, 0.7],
    "criterion": ["mse", "mae"],
}
grid = GridSearchCV(RandomForestRegressor(),params, scoring=make_scorer(mean_absolute_error, greater_is_better=False) )
X_train, X_test, y_train, y_test = train_test_split(train_processed, target)
%%time
grid.fit(X_train, y_train)
clf = grid.best_estimator_; clf
clf.fit(X_train, y_train)
mean_absolute_error(clf.predict(X_test), y_test)<jupyter_output><empty_output><jupyter_text>LightGBM<jupyter_code>params = {
        "max_depth": [-1, 10, 20, 50],
        "num_leaves": [5,13, 20, 40, 50], 
        "learning_rate": [0.01, 0.001, 0.1],
        "bagging_freq": [5, 10, 3, 15], 
        "bagging_fraction": [0.4, 0.2, 0.6], 
        "feature_fraction":[0.05, 0.01, 0.03], 
        "min_data_in_leaf":[20, 40, 80, 100], 
        "min_sum_heassian_in_leaf": [5, 10, 15, 20, 40], 
}
grid = GridSearchCV(LGBMRegressor(), params, scoring=make_scorer(mean_absolute_error, greater_is_better=False))
%%time
grid.fit(X_train, y_train)
clf = grid.best_estimator_; clf
clf.fit(X_train, y_train)
mean_absolute_error(clf.predict(X_test), y_test)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Regressor<jupyter_code>params = {
    "n_estimators": [1, 20, 50, 100, 200, 500],
    "max_depth": [3, 10, 15, 20],
    "max_features": ["auto", "sqrt", "log2", 0.5, 0.7],
    "criterion": ["mse", "mae"],
}
grid = GridSearchCV(GradientBoostingRegressor(), params, scoring=make_scorer(mean_absolute_error, greater_is_better=False))
%%time
grid.fit(X_train, y_train)
clf = grid.best_estimator_; clf

clf.fit(X_train, y_train)
mean_absolute_error(clf.predict(X_test), y_test)
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.model_selection import KFold
from sklearn.tree import DecisionTreeRegressor
from sklearn.base import clone
import numpy as np
gbm = GradientBoostingRegressor(alpha=0.9, criterion='mae', init=None,
                              learning_rate=0.1, loss='ls', max_depth=3, max_features=0.7,
                              max_leaf_nodes=None, min_impurity_decrease=0.0,
                              min_impurity_split=None, min_samples_leaf=1,
                              min_samples_split=2, min_weight_fraction_leaf=0.0,
                              n_estimators=20, n_iter_no_change=None, presort='auto',
                              random_state=None, subsample=1.0, tol=0.0001,
                              validation_fraction=0.1, verbose=0, warm_start=False)
rf = RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=10,
                          max_features='auto', max_leaf_nodes=None,
                          min_impurity_decrease=0.0, min_impurity_split=None,
                          min_samples_leaf=1, min_samples_split=2,
                          min_weight_fraction_leaf=0.0, n_estimators=50, n_jobs=None,
                          oob_score=False, random_state=None, verbose=0, warm_start=False),
models = [
    
    AdaBoostRegressor(base_estimator=DecisionTreeRegressor(criterion='mse', max_depth=3, 
                                                           max_features=None,max_leaf_nodes=None, min_impurity_decrease=0.0,
                                                           min_impurity_split=None, min_samples_leaf=1,
                                                           min_samples_split=2, min_weight_fraction_leaf=0.0, 
                                                           presort=False, random_state=None, splitter='best'),
                      learning_rate=1, loss='exponential', n_estimators=25,
                      random_state=None),
    XGBRegressor(base_score=0.5, booster='dart', colsample_bylevel=1,
                 colsample_bytree=1, gamma=0, importance_type='gain',
                 learning_rate=0.1, max_delta_step=0, max_depth=2,
                 min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,
                 nthread=None, objective='reg:linear', random_state=0, reg_alpha=0,
                 reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,
                 subsample=1),
    Ridge(alpha=0.75, copy_X=True, fit_intercept=True, max_iter=None,
          normalize=False, random_state=None, solver='sparse_cg', tol=0.001),
    Lasso(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=1000,
          normalize=False, positive=False, precompute=False, random_state=None,
          selection='cyclic', tol=0.0001, warm_start=False),
    gbm,
    LGBMRegressor(bagging_fraction=0.6, bagging_freq=3, boosting_type='gbdt',
                  class_weight=None, colsample_bytree=1.0, feature_fraction=0.05,
                  importance_type='split', learning_rate=0.1, max_depth=-1,
                  min_child_samples=20, min_child_weight=0.001, min_data_in_leaf=20,
                  min_split_gain=0.0, min_sum_heassian_in_leaf=5, n_estimators=100,
                  n_jobs=-1, num_leaves=5, objective=None, random_state=None,
                  reg_alpha=0.0, reg_lambda=0.0, silent=True, subsample=1.0,
                  subsample_for_bin=200000, subsample_freq=0),
    # Add GBM later, since it is taking a lot of time
    
    
    
]
test_df = pd.read_csv(f"{PATH}/bodyfat-test.csv", index_col="Id")
test_df = feature_engineering_create_ratios(test_df)
l = list(test_df.columns)
train_df = pd.read_csv(f"{PATH}/bodyfat-train.csv")
test_df = pd.read_csv(f"{PATH}/bodyfat-test.csv", index_col="Id")

train_df = remove_outliers(train_df)


preds = np.zeros(subm_df.shape[0])
n_splits = 4

# train_X, test_X, train_y, test_y = train_test_split(train_processed, target, random_state=42, test_size=0.2)
# train_X, test_X, train_y, test_y = train_test_split(train_processed, target, test_size=0.25)
d = {}
kfold = KFold(n_splits=n_splits, shuffle=True)
test_df = feature_engineering_create_ratios(test_df)
for i, (train_idx, test_idx) in enumerate(kfold.split(train_df)):
    train_processed = row_augmentation(train_df.iloc[train_idx], num_aug=5)
    target = train_processed["Body fat (%)"]
    train_processed = train_processed.drop(["Body fat (%)"], axis=1)
    train_processed = feature_engineering_create_ratios(train_processed)
    X_train = train_processed
    y_train = target
    X_test = train_df.iloc[test_idx]
    y_test = X_test["Body fat (%)"]
    X_test = X_test.drop(["Body fat (%)"], axis=1)
    X_test = feature_engineering_create_ratios(X_test)
    valid_preds = np.zeros(X_test.shape[0])
    for model in models:
        #break
        name = model.__str__().split("(")[0]
        print(f"Training {name} at {i} fold")
        d[name] = clone(model)
        d[name].fit(X_train, y_train)
        print(f"Training {name} done")
        pred = d[name].predict(X_test)
        valid_preds += pred
        r2 = r2_score(pred, y_test)
        mas = mean_absolute_error(pred, y_test)
        msle=0
        preds += d[name].predict(test_df)
    #     msle = mean_squared_log_error(pred, test_y)
        print(f"r2: {r2}, msle: {msle}, mas: {mas}, {name}")
        d[name] = [d[name], r2, mas]
        

    # NN pred
    y_test = pd.DataFrame(y_test)
    y_train = pd.DataFrame(y_train)
    
    X_train = torch.Tensor(X_train.values)
    X_test = torch.Tensor(X_test.values)
    y_train = torch.Tensor(y_train.values)
    y_test = torch.Tensor(y_test.values)

    train_dataset = TensorDataset(X_train, y_train)
    test_dataset = TensorDataset(X_test, y_test)

    train_loader = DataLoader(train_dataset, batch_size=50, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=50)

    model = get_nn_model()
    
    optim = torch.optim.ASGD(model.parameters(), lr=1e-6, weight_decay=0.1)

    criterion = nn.MSELoss()

    losses = []
    for epoch in range(600):
        for X, y in train_loader:
            optim.zero_grad()
            X_v = Variable(X)
            y_v = Variable(y)
            y_pred = model(X_v)
            loss = criterion(y_pred, y_v)
            losses.append(loss.data)
            loss.backward()
            optim.step()
    #         print(loss.data[0])

        for X, y in test_loader:
            X_v = Variable(X)
            y_v = Variable(y) 
            y_pred = model(X_v)
            npp = y_pred.data.numpy()
            npy = y.numpy()
            score = mean_absolute_error(npp, npy)
            if epoch % 100 == 0:
                print(f"Epoch: {epoch} Score: {score}")
            
    x_test = torch.Tensor(test_df.values)
    x_test = Variable(x_test)
    y_pred = model(x_test)
    y_pred = y_pred.data.numpy()
    y_pred = y_pred.reshape(preds.shape)
    print(y_pred.shape, preds.shape)
    preds += y_pred
    

        
        
        
    print(f"Split {i} done ..........")
    print("Scores:")
    for name in d.keys():
        print(f"name: {name} score: {d[name][2]}")
    
    ensembled_preds = valid_preds/len(models)
    ensembled_r2 = r2_score(ensembled_preds, y_test)
    ensembled_mas = mean_absolute_error(ensembled_preds, y_test)
    print(f"Ensembled socre: r2: {ensembled_r2}, mas: {ensembled_mas}")
    
len(models)
preds = preds/((len(models) + 1)*n_splits)
preds
pred
subm_df = pd.read_csv("../data/raw/kaggle_solution_template.csv")
subm_df.head()
subm_df.shape
subm_df["Body fat(%)"] = preds
subm_df.head()
subm_df.to_csv("../data/raw/submission_row_augmented_ratio_augmented_7_models.csv", index=None)
train_processed = feature_engineering(train_df.drop(['Body fat (%)'], axis=1))<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>import torch
from torch.utils.data import Dataset, DataLoader, TensorDataset
from torch import Tensor
from torch import nn
from torch.autograd import Variable
class LinearRegression(nn.Module):
    def __init__(self, in_channels, hidden_size):
        super(LinearRegression, self).__init__()
        self.l0 = nn.Linear(in_channels, hidden_size[0])
        self.d0 = nn.Dropout(p=0.2)
        self.l1 = nn.Linear(hidden_size[0], hidden_size[1])
        self.d1 = nn.Dropout(p=0.2)
        self.l2 = nn.Linear(hidden_size[1], hidden_size[2])
        self.d2 = nn.Dropout(p=0.2)
        self.l3 = nn.Linear(hidden_size[2], 1)

    def forward(self, x):
        return self.l3(self.d2(self.l2(self.d1(self.l1(self.d0(self.l0((x))))))))
    
def get_nn_model():
    return LinearRegression(in_channels=train_dataset[0][0].size()[0], hidden_size=[128,64,16]) 
train_df = pd.read_csv(f"{PATH}/bodyfat-train.csv")
test_df = pd.read_csv(f"{PATH}/bodyfat-test.csv", index_col="Id")

train_df = remove_outliers(train_df)

preds = np.zeros(subm_df.shape[0])
n_splits = 4

# train_X, test_X, train_y, test_y = train_test_split(train_processed, target, random_state=42, test_size=0.2)
# train_X, test_X, train_y, test_y = train_test_split(train_processed, target, test_size=0.25)
d = {}
kfold = KFold(n_splits=n_splits, shuffle=True)
#test_df = feature_engineering_create_ratios(test_df)
for i, (train_idx, test_idx) in enumerate(kfold.split(train_df)):
    
    # Get train & test df 
    train_fold_df = train_df.iloc[train_idx]
    test_fold_df = train_df.iloc[test_idx]
    
    X_train = train_fold_df.drop(["Body fat (%)"],axis=1)
    y_train = train_fold_df["Body fat (%)"]
    
    X_test = test_fold_df.drop(["Body fat (%)"],axis=1)
    y_test = test_fold_df["Body fat (%)"]
    
    
    y_train = pd.DataFrame(y_train); y_test = pd.DataFrame(y_test)

    X_train["Body fat (%)"] = y_train

    train_processed = row_augmentation(X_train, num_aug=5)
    y_train = pd.DataFrame(train_processed["Body fat (%)"])
    train_processed = train_processed.drop(["Body fat (%)"], axis=1)
    train_processed = feature_engineering_create_ratios(train_processed)
    X_train = train_processed
    X_test = feature_engineering_create_ratios(X_test)

    X_train = torch.Tensor(X_train.values)
    X_test = torch.Tensor(X_test.values)
    y_train = torch.Tensor(y_train.values)
    y_test = torch.Tensor(y_test.values)

    train_dataset = TensorDataset(X_train, y_train)
    test_dataset = TensorDataset(X_test, y_test)

    train_loader = DataLoader(train_dataset, batch_size=50, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=50)

    model = get_nn_model()
    
    optim = torch.optim.ASGD(model.parameters(), lr=1e-6, weight_decay=0.1)

    criterion = nn.MSELoss()

    losses = []
    for epoch in range(600):
        for X, y in train_loader:
            optim.zero_grad()
            X_v = Variable(X)
            y_v = Variable(y)
            y_pred = model(X_v)
            loss = criterion(y_pred, y_v)
            losses.append(loss.data)
            loss.backward()
            optim.step()
    #         print(loss.data[0])

        for X, y in test_loader:
            X_v = Variable(X)
            y_v = Variable(y) 
            y_pred = model(X_v)
            npp = y_pred.data.numpy()
            npy = y.numpy()
            score = mean_absolute_error(npp, npy)
            print(f"Epoch: {epoch} Score: {score}")


    print(f"Split {i} done ..........")
    print("Scores:")


plt.plot(losses)
print(losses[10])<jupyter_output>tensor(300.8222)
{'fraction_non_alphanumeric': 0.12542028801624056, 'fraction_numerical': 0.02436084501681152, 'mean_word_length': 3.310637134263057, 'pattern_counts': {'":': 16, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '135276', 'license': 'MIT', 'n_tokens_mistral': 5940, 'n_tokens_neox': 5577, 'n_words': 968, 'path': 'notebooks/udacity-body-fat-prediction-challenge.ipynb', 'repo_name': 'manishym/udacity-body-fat-prediction-challenge', 'size': '15763'}