<jupyter_start><jupyter_text>Question 1<jupyter_code>df_2.isnull().sum()<jupyter_output><empty_output><jupyter_text>Question 2<jupyter_code>df_2.minimum_nights.median()

n = len(df_2)

n_val = int(n * 0.2)
n_test = int(n * 0.2)
n_train = n - n_val - n_test

idx = np.arange(n)

np.random.seed(42)
np.random.shuffle(idx)

df_train      = df_2.iloc[idx[:n_train]]
df_val        = df_2.iloc[idx[n_train:n_train+n_val]]
df_test       = df_2.iloc[idx[n_train+n_val:]]
df_train_test = df_2.iloc[idx[:n_train+n_test]]

y_train = np.log1p(df_train.price)
y_val   = np.log1p(df_val.price)
y_test  = np.log1p(df_test.price)

del df_train['price']
del df_val['price']
del df_test['price']
len(y_train)<jupyter_output><empty_output><jupyter_text>Question 3<jupyter_code>def prepare_X_zero(df):
    df = df.copy()
    
    df['reviews_per_month'] = df['reviews_per_month'].fillna(0)
    X = df.values

    return X
X_train_zero = prepare_X_zero(df_train)
X_val_zero   = prepare_X_zero(df_val)
def prepare_X_mean(df):
    df = df.copy()
    mean_value = df['reviews_per_month'].mean()
    df['reviews_per_month'] = df['reviews_per_month'].fillna(mean_value)
    X = df.values

    return X
X_train_mean = prepare_X_mean(df_train)
X_val_mean   = prepare_X_mean(df_val)
def train_linear_regression(X, y):
    ones = np.ones(X.shape[0])
    X = np.column_stack([ones, X])
    XTX = X.T.dot(X)
    XTX_inv = np.linalg.inv(XTX)
    w_full = XTX_inv.dot(X.T).dot(y)
    return w_full[0], w_full[1:]
w0_zero, w_zero = train_linear_regression(X_train_zero, y_train)
w0_mean, w_mean = train_linear_regression(X_train_mean, y_train)
X_val_zero = prepare_X_zero(df_val)
X_val_mean = prepare_X_mean(df_val)
y_pred_zero = w0_zero + X_val_zero.dot(w_zero)
y_pred_mean = w0_mean + X_val_mean.dot(w_mean)
#Use the validation dataset to evaluate the models and compare the RMSE of each option.

def rmse(y, y_pred):
    se = (y - y_pred) ** 2
    mse = se.mean()
    return np.sqrt(mse)

rmse(y_val, y_pred_zero)
rmse(y_val, y_pred_mean)
print(rmse(y_val, y_pred_zero))
print(rmse(y_val, y_pred_mean))

score_zero = rmse(y_val, y_pred_zero)
round(score_zero, 2)
print(round(score_zero, 2))
score_mean = rmse(y_val, y_pred_mean)
round(score_mean, 2)
print(round(score_mean, 2))<jupyter_output>0.64<jupyter_text>Question 4<jupyter_code>#Now let's train a regularized linear regression.
#For this question, fill the NAs with 0.
#Try different values of r from this list: [0, 0.000001, 0.0001, 0.001, 0.01, 0.1, 1, 5, 10].
#Use RMSE to evaluate the model on the validation dataset.
#Round the RMSE scores to 2 decimal digits.
#Which r gives the best RMSE?
def train_linear_regression_reg(X, y, r=0.001):
    ones = np.ones(X.shape[0])
    X = np.column_stack([ones, X])

    XTX = X.T.dot(X)
    XTX = XTX + r * np.eye(XTX.shape[0])

    XTX_inv = np.linalg.inv(XTX)
    w_full = XTX_inv.dot(X.T).dot(y)
    
    return w_full[0], w_full[1:]

prepare_X = prepare_X_zero
for r in [0.0, 0.00001, 0.0001, 0.001, 0.1, 1, 10]:
    
    X_train = prepare_X(df_train)
    w0, w = train_linear_regression_reg(X_train, y_train, r=r)

    X_val = prepare_X(df_val)
    y_pred = w0 + X_val.dot(w)
    score = rmse(y_val, y_pred)
    
    score = round(score, 2)
    
    
    print(r, w0, score)<jupyter_output>0.0 -419.9126587294167 0.64
1e-05 -419.4137637716231 0.64
0.0001 -414.97649243207684 0.64
0.001 -375.2736527030572 0.64
0.1 -32.56256055226638 0.68
1 -3.499216837729211 0.68
10 -0.35127675916757617 0.68<jupyter_text>Question 5<jupyter_code>#We used seed 42 for splitting the data. Let's find out how selecting the seed influences our score.
#Try different seed values: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].
#For each seed, do the train/validation/test split with 60%/20%/20% distribution.
#Fill the missing values with 0 and train a model without regularization.
#For each seed, evaluate the model on the validation dataset and collect the RMSE scores.
#What's the standard deviation of all the scores? To compute the standard deviation, use np.std.
#Round the result to 3 decimal digits (round(std, 3))
errors = []
for seed in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]:
    idx = np.arange(n)

    np.random.seed(seed)
    np.random.shuffle(idx)

    df_train      = df_2.iloc[idx[:n_train]]
    df_val        = df_2.iloc[idx[n_train:n_train+n_val]]

    y_train = np.log1p(df_train.price)
    y_val   = np.log1p(df_val.price)

    del df_train['price']
    del df_val['price']
    
    X_train = prepare_X(df_train)
    w0, w = train_linear_regression_reg(X_train, y_train, r=0)

    X_val = prepare_X(df_val)
    y_pred = w0 + X_val.dot(w)
    score = rmse(y_val, y_pred)
    errors.append(score)
np.std(errors)

print(round(np.std(errors), 3))<jupyter_output>0.008<jupyter_text>Question 6<jupyter_code>#Split the dataset like previously, use seed 9.
#Combine train and validation datasets.
#Fill the missing values with 0 and train a model with r=0.001.
#What's the RMSE on the test dataset?


    n = len(df_2)

    n_val = int(n * 0.2)
    n_test = int(n * 0.2)
    n_train = n - n_val - n_test

    idx = np.arange(n)

    np.random.seed(9)
    np.random.shuffle(idx)

    df_train      = df_2.iloc[idx[:n_train]]
    df_val        = df_2.iloc[idx[n_train:n_train+n_val]]
    df_test       = df_2.iloc[idx[n_train+n_val:]]

    df_combo = df_2.iloc[idx[:n_train+n_val]]

    y_train = np.log1p(df_train.price)
    y_val   = np.log1p(df_val.price)
    y_test  = np.log1p(df_test.price)

    del df_train['price']
    del df_val['price']
    del df_test['price']
    
    prepare_X = prepare_X_zero
    
    X_train = prepare_X(df_train)
    w0, w = train_linear_regression_reg(X_train, y_train, r=0.01)

    X_val = prepare_X(df_val)
    y_pred = w0 + X_val.dot(w)
    score = rmse(y_val, y_pred)
    
    score = round(score, 2)
    
    
    print(r, w0, score)<jupyter_output>10 -193.96323221220518 0.65
{'fraction_non_alphanumeric': 0.1296928327645051, 'fraction_numerical': 0.059044368600682594, 'mean_word_length': 3.819901315789474, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '260923', 'license': 'CC0-1.0', 'n_tokens_mistral': 2736, 'n_tokens_neox': 2496, 'n_words': 519, 'path': 'mlzoomcamp/wk02/wk02_hw.ipynb', 'repo_name': 'jazwilson/workbook', 'size': '5860'}