<jupyter_start><jupyter_text>Impute missing values with "ExtraTreesRegressor"<jupyter_code># Import pandas
import pandas as pd
import pytoolbox as ptb
import numpy as np
import matplotlib.pyplot as plt
import datetime as dt
import warnings

warnings.simplefilter(action='ignore', category=FutureWarning)
# Upload 'bike_rentals.csv' to dataFrame
df_bikes = pd.read_csv('bike_rentals.csv')
df_bikes.groupby(['season']).agg(['median'])
idx = df_bikes.loc[df_bikes.isnull().any(axis=1),df_bikes.isnull().any()].index
print(idx)
df_bikes.loc[df_bikes.isnull().any(axis=1),df_bikes.isnull().any()]
# impute missing values

# Imputations I
for i in ['temp', 'atemp', 'hum', 'windspeed']:
    df_bikes[i].fillna(df_bikes.groupby('season')[i].transform('median'), inplace=True)

# Imputations II
df_bikes['dteday'] = pd.to_datetime(df_bikes['dteday'], infer_datetime_format=True)
df_bikes['mnth'] = df_bikes['dteday'].dt.month

# Imputations III
df_bikes.loc[730, 'yr'] = 1.0
df_bikes.iloc[idx]
df_bikes[df_bikes.isnull().any(axis=1)]
# Drop 'casual', 'registered' columns
df_bikes = df_bikes.drop(['dteday', 'casual', 'registered'], axis=1)
# Split data into X and y
X = df_bikes.iloc[:,:-1]
y = df_bikes.iloc[:,-1]
# Import train_test_split
from sklearn.model_selection import train_test_split

# Import Linear Regression
from sklearn.linear_model import LinearRegression

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=2)<jupyter_output><empty_output><jupyter_text>Impute missing values<jupyter_code>from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
from sklearn.impute import KNNImputer

# impute_estimator = KNeighborsRegressor(n_neighbors=5)
impute_estimator = ExtraTreesRegressor(n_estimators=10, random_state=0)
# impute_estimator = RandomForestRegressor(n_estimators=10, random_state=0)

imp_mean = IterativeImputer(random_state=0, estimator=impute_estimator)

# impute_estimator = KNNImputer(n_neighbors=5)
X_train_imputed = imp_mean.fit_transform(X_train)
X_test_imputed = imp_mean.transform(X_test)

# train/test approach
xg_reg = XGBRegressor()
xg_reg.fit(X_train_imputed, y_train)
y_pred = xg_reg.predict(X_test_imputed)
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
print('RMSE: {0:0.2f}\n'.format(rmse))

# k-fold cross validation
model = XGBRegressor(objective="reg:squarederror")
X_imputed = df_bikes.iloc[:,:-1]
X_imputed = imp_mean.fit_transform(X_imputed)
scores = cross_val_score(model, X_imputed, y, scoring='neg_mean_squared_error', cv=10)
rmse = np.sqrt(-scores)
print('Reg rmse:', np.round(rmse, 2))
print('RMSE mean: %0.2f' % (rmse.mean()))
print('RMSE std: {0:0.2f}'.format(rmse.std()))
df_imputed = pd.read_csv('bike_rentals_cleaned.csv')
df_imputed = pd.DataFrame(X_imputed, columns=df_imputed.columns[:-1])

plt.style.use('ggplot')
df_imputed.loc[:, df_imputed.columns!='instant'].hist(bins=50, figsize=(20,15), ec='black', color='#3380FF')
plt.show()<jupyter_output><empty_output><jupyter_text>XGB book<jupyter_code>df_bikes = pd.read_csv('bike_rentals_cleaned.csv')
X = df_bikes.iloc[:,:-1]
y = df_bikes.iloc[:,-1]
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=2)
# LinearRegression model
lin_reg = LinearRegression()
lin_reg.fit(X_train, y_train)
y_pred = lin_reg.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
print("RMSE: %0.2f" % (rmse))
xg_reg = XGBRegressor()
xg_reg.fit(X_train, y_train)
y_pred = xg_reg.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
print("RMSE: %0.2f" % (rmse))
model = LinearRegression()
scores = cross_val_score(model, X, y, scoring='neg_mean_squared_error', cv=10)
rmse = np.sqrt(-scores)
print('Reg rmse:', np.round(rmse, 2))
print('RMSE mean: %0.2f' % (rmse.mean()))
model = XGBRegressor(objective="reg:squarederror")
scores = cross_val_score(model, X, y, scoring='neg_mean_squared_error', cv=10)
rmse = np.sqrt(-scores)
print('Reg rmse:', np.round(rmse, 2))
print('RMSE mean: %0.2f' % (rmse.mean()))
print('RMSE std: {0:0.2f}'.format(rmse.std()))<jupyter_output>Reg rmse: [ 717.65  692.8   520.7   737.68  835.96 1006.24  991.34  747.61  891.99
 1731.13]
RMSE mean: 887.31
RMSE std: 313.46<jupyter_text>Machine Learning - Classification<jupyter_code># Upload Census dataset (adult) from UCI Machine Learning Repository
df_census = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data')

# Display first 5 rows
df_census.head()
# Upload Census dataset with no header
df_census = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data', header=None)

# Display first 5 rows
df_census.head()
# Define df_census columns
df_census.columns = ['age', 'workclass', 'fnlwgt', 'education', 'education-num', 'marital-status', 'occupation',
                  'relationship', 'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week', 'native-country', 
                   'income']

# Display first 5 rows
df_census.head()
# Display df_census info
df_census.info()
# Drop education column
df_census = df_census.drop(['education'], axis=1)
df_omd = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data', header=None)
df_omd.columns = ['age', 'workclass', 'fnlwgt', 'education', 'education-num', 'marital-status', 'occupation',
                  'relationship', 'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week', 'native-country', 
                   'income']
df_omd = df_omd.drop(['education'], axis=1)
df_omd.head()
from sklearn.preprocessing import OneHotEncoder

enc = OneHotEncoder(handle_unknown = 'ignore')
enc.fit(df_omd)

df_omd = enc.transform(df_omd)
df_omd
print(df_omd[1:13, :])
# Convert non-numeric columns using get_dummies
df_census = pd.get_dummies(df_census)

# Display first 5 rows
df_census.head()
# Drop column 'income_ <=50K'
df_census = df_census.drop('income_ <=50K', axis=1)
# Split data into X and y
X = df_census.iloc[:,:-1]
y = df_census.iloc[:,-1]
# Import Logistic Regression
from sklearn.linear_model import LogisticRegression
# Import cross_val_score
from sklearn.model_selection import cross_val_score
from sklearn.metrics import roc_auc_score

# Define cross_val function with classifer and num_splits as input
def cross_val(classifier, num_splits=10):
    
    # Initialize classifier
    model = classifier

    # Obtain scores of cross-validation
    scores = cross_val_score(model, X, y, cv=num_splits)
    roc = cross_val_score(model, X, y, scoring="roc_auc", cv=num_splits)

    # Display accuracy
    print('Accuracy:', np.round(scores, 2))
    print('ROC: {}'.format(roc))

    # Display mean accuracy
    print('Accuracy mean: %0.2f' % (scores.mean()))
    print('ROC mean: {0:0.2f}'.format((roc.mean())))
# Use cross_val function to score LogisticRegression
cross_val(LogisticRegression())
# Import XGBoost Classifier
from xgboost import XGBClassifier
# Use cross_val function to score XGBoost
cross_val(XGBClassifier(n_estimators=100))
import numpy as np
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.metrics import auc
from sklearn.metrics import RocCurveDisplay
from sklearn.model_selection import StratifiedKFold

# #############################################################################
# Data IO and generation

# Import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
X, y = X[y != 2], y[y != 2]
n_samples, n_features = X.shape

# Add noisy features
random_state = np.random.RandomState(0)
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]

# #############################################################################
# Classification and ROC analysis

# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=6)
classifier = svm.SVC(kernel="linear", probability=True, random_state=random_state)

tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)

fig, ax = plt.subplots()
for i, (train, test) in enumerate(cv.split(X, y)):
    classifier.fit(X[train], y[train])
    viz = RocCurveDisplay.from_estimator(
        classifier,
        X[test],
        y[test],
        name="ROC fold {}".format(i),
        alpha=0.3,
        lw=1,
        ax=ax,
    )
    interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)
    interp_tpr[0] = 0.0
    tprs.append(interp_tpr)
    aucs.append(viz.roc_auc)

ax.plot([0, 1], [0, 1], linestyle="--", lw=2, color="r", label="Chance", alpha=0.8)

mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
ax.plot(
    mean_fpr,
    mean_tpr,
    color="b",
    label=r"Mean ROC (AUC = %0.2f $\pm$ %0.2f)" % (mean_auc, std_auc),
    lw=2,
    alpha=0.8,
)

std_tpr = np.std(tprs, axis=0)
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
ax.fill_between(
    mean_fpr,
    tprs_lower,
    tprs_upper,
    color="grey",
    alpha=0.2,
    label=r"$\pm$ 1 std. dev.",
)

ax.set(
    xlim=[-0.05, 1.05],
    ylim=[-0.05, 1.05],
    title="Receiver operating characteristic example",
)
ax.legend(loc="lower right")
plt.show()
from sklearn.impute import KNNImputer
import numpy as np

X = [ [3, np.NaN], [1, 2], [5, 10], [3, 6], [4, 8] ]
X_train = [[1, 2], [16, np.NaN], [25, 50], [50, 100], [3, 6] ]
print("X: ", X)
print("===========")


imputer = KNNImputer(n_neighbors= 1)
impute_with_1 = imputer.fit_transform(X)

print("\nImpute with 1 Neighbour: \n", impute_with_1)



imputer = KNNImputer(n_neighbors= 2)
impute_with_2 = imputer.fit_transform(X)

print("\n Impute with 2 Neighbours: \n", impute_with_1)
imputer = KNNImputer(n_neighbors= 2)
imputer.fit(X)
impute_omd = imputer.transform(X_train)
print(X_train)
print(impute_omd)
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import RandomForestRegressor

impute_estimator = ExtraTreesRegressor(n_estimators=10, random_state=0)
# impute_estimator = RandomForestRegressor(n_estimators=10, random_state=0)
imp_mean = IterativeImputer(random_state=0, estimator=impute_estimator)
imp_mean.fit(X)
impute_omd = imp_mean.transform(X_train)
print(impute_omd)
from sklearn.preprocessing import OneHotEncoder

### Correct
train = pd.DataFrame(['A', 'B', 'A', 'C'])
test = pd.DataFrame(['B', 'A', 'D'])

enc = OneHotEncoder(handle_unknown = 'ignore')
enc.fit(train)

enc.transform(train).toarray()
#array([[1., 0., 0.],
#       [0., 1., 0.],
#       [1., 0., 0.],
#       [0., 0., 1.]])

enc.transform(test).toarray()
#array([[0., 1., 0.],
#       [1., 0., 0.],
#       [0., 0., 0.]])

### Incorrect
full = pd.concat((train, test))

enc = OneHotEncoder(handle_unknown = 'ignore')
enc.fit(full)

enc.transform(train).toarray()
#array([[1., 0., 0., 0.],
#       [0., 1., 0., 0.],
#       [1., 0., 0., 0.],
#       [0., 0., 1., 0.]])

enc.transform(test).toarray()
#array([[0., 1., 0., 0.],
#       [1., 0., 0., 0.],
#       [0., 0., 0., 1.]])
train
enc.transform(train).toarray()
test
enc.transform(test).toarray()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16454482036972445, 'fraction_numerical': 0.0257237530519707, 'mean_word_length': 5.287828947368421, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '426694', 'license': 'MIT', 'n_tokens_mistral': 4715, 'n_tokens_neox': 4428, 'n_words': 898, 'path': 'Chapter01/omd_impute_missing_values_xgb.ipynb', 'repo_name': 'omdgit/Hands-On-Gradient-Boosting-with-XGBoost-and-Scikit-learn', 'size': '11468'}