<jupyter_start><jupyter_text>Import libraries<jupyter_code>import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.metrics import mutual_info_score
from sklearn.feature_extraction import DictVectorizer
from sklearn.model_selection import train_test_split


from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import export_text
from sklearn.ensemble import RandomForestRegressor<jupyter_output><empty_output><jupyter_text>Import and prepare data<jupyter_code>data = 'https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2019/2019-03-12/board_games.csv'
!curl $data > data.csv
df_original = pd.read_csv('data.csv')
df_original.head()
df = df_original[[
           'max_players',
           'max_playtime',
           'min_age',
           'min_players',
           'min_playtime',
           'playing_time',
           'year_published',
           'category',
           'mechanic',
           'average_rating',
           'users_rated']]

df.dtypes
categorical_columns   = [
           'name',           
           'category',
           'mechanic']<jupyter_output><empty_output><jupyter_text>Exploratory data analysis<jupyter_code>%matplotlib inline
df.average_rating.mean()
sns.histplot(df.average_rating, bins=50)
sns.histplot(df.year_published, bins=50)
user_logs = np.log1p(df.users_rated)
sns.histplot(user_logs, bins=50)
PT_logs = np.log1p(df.playing_time)
sns.histplot(PT_logs, bins=50)<jupyter_output><empty_output><jupyter_text>Missing values<jupyter_code>df.isnull().sum()
df = df.fillna(method="ffill")
df.isnull().sum()
df.head()
df.shape
df.iloc[29]<jupyter_output><empty_output><jupyter_text>Encode categorical features   Category variable eg. 'Adventure', 'Educational', 'Puzzle'<jupyter_code># Get a list of the categories
category   = sorted(set(",".join(df.category).split(",")))

# Convert to binary readout: 0 = not category, 1 = category 
for c in category:
    df[c] = df.category.str.contains(c).astype(int)
    
category<jupyter_output><empty_output><jupyter_text>Mechanic variable eg. 'Campaign', 'Player Elimination', 'Commodity Speculation'<jupyter_code>mechanic   = sorted(set(",".join(df.mechanic).split(",")))
for m in mechanic:
    df[m] = df.mechanic.str.contains(m).astype(int)

mechanic
df.shape
game_col = list(df.dtypes[df.dtypes == 'int64'].index)
game_col<jupyter_output><empty_output><jupyter_text>Feature importance: Mutual information<jupyter_code>def mutual_info_av_rating_score(series):
    return mutual_info_score(series, df.average_rating > np.median(df.average_rating))
mi = df[game_col].apply(mutual_info_av_rating_score).round(2)
mi.sort_values(ascending=False)<jupyter_output><empty_output><jupyter_text>Feature importance: Correlation<jupyter_code>corr = df[game_col].corr()
corr
plt.figure(figsize = (30,30))
sns.heatmap(corr, xticklabels=corr.columns, yticklabels=corr.columns, cmap=sns.diverging_palette(220, 10, as_cmap=True))<jupyter_output><empty_output><jupyter_text>Setting up the validation framework  Split data<jupyter_code>df_full_train, df_test = train_test_split(df, test_size=0.2, random_state=42)
df_train, df_val = train_test_split(df_full_train, test_size=0.25, random_state=42)

len(df_train), len(df_val), len(df_test)

df_train = df_train.reset_index(drop=True)
df_val   = df_val.reset_index(drop=True)
df_test  = df_test.reset_index(drop=True)

y_full_train = df_full_train.average_rating.values
y_train = df_train.average_rating.values
y_val   = df_val.average_rating.values
y_test  = df_test.average_rating.values

del df_full_train['average_rating']
del df_train['average_rating']
del df_val['average_rating']
del df_test['average_rating']
dv = DictVectorizer(sparse=False)

train_dict = df_train[game_col].to_dict(orient='records')
X_train = dv.fit_transform(train_dict)

val_dict = df_val[game_col].to_dict(orient='records')
X_val = dv.transform(val_dict)<jupyter_output><empty_output><jupyter_text>Models  DecisionTreeRegressor  Define rmse<jupyter_code>def rmse(y, y_pred):
    se = (y - y_pred) ** 2
    mse = se.mean()
    return np.sqrt(mse)<jupyter_output><empty_output><jupyter_text>Depth optimisation<jupyter_code>depths = [1, 2, 3, 4, 5, 6, 10, 15, 20, None]

for depth in depths: 
    dt = DecisionTreeRegressor(max_depth=depth)
    dt.fit(X_train, y_train)
    
    y_pred = dt.predict(X_val)
    error = rmse(y_val, y_pred)
    
    print('%4s -> %.3f' % (depth, error))<jupyter_output>1 -> 0.782
   2 -> 0.738
   3 -> 0.697
   4 -> 0.667
   5 -> 0.638
   6 -> 0.626
  10 -> 0.680
  15 -> 0.741
  20 -> 0.779
None -> 0.813<jupyter_text>Depth of 4, 5 and 6 selected as these had the lowest error values  Min_samples_leaf optimisation<jupyter_code>scores = []

for depth in [4, 5, 6]:
    for s in [1, 5, 10, 15, 20, 500, 100, 200]:
        dt = DecisionTreeRegressor(max_depth=depth, min_samples_leaf=s)
        dt.fit(X_train, y_train)
    
        y_pred = dt.predict(X_val)
        error = rmse(y_val, y_pred)
        
        scores.append((depth, s, error))
columns = ['max_depth', 'min_samples_leaf', 'error']
df_scores = pd.DataFrame(scores, columns=columns)

df_scores_pivot = df_scores.pivot(index='min_samples_leaf', columns=['max_depth'], values=['error'])
df_scores_pivot.round(3)<jupyter_output><empty_output><jupyter_text>Plot error for max_depth and min_samples_leaf parameters<jupyter_code>sns.heatmap(df_scores_pivot, annot=True, fmt=".3f")<jupyter_output><empty_output><jupyter_text>Select optimal DecisionTreeRegressor parameters<jupyter_code>dt = DecisionTreeRegressor(max_depth=6, min_samples_leaf=1)
dt.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Print feature names<jupyter_code>print(export_text(dt, feature_names=dv.get_feature_names()))<jupyter_output>|--- year_published <= 2009.50
|   |--- max_playtime <= 135.00
|   |   |--- users_rated <= 801.50
|   |   |   |--- Wargame <= 0.50
|   |   |   |   |--- Roll / Spin and Move <= 0.50
|   |   |   |   |   |--- users_rated <= 105.50
|   |   |   |   |   |   |--- value: [5.70]
|   |   |   |   |   |--- users_rated >  105.50
|   |   |   |   |   |   |--- value: [5.98]
|   |   |   |   |--- Roll / Spin and Move >  0.50
|   |   |   |   |   |--- users_rated <= 137.50
|   |   |   |   |   |   |--- value: [5.26]
|   |   |   |   |   |--- users_rated >  137.50
|   |   |   |   |   |   |--- value: [5.59]
|   |   |   |--- Wargame >  0.50
|   |   |   |   |--- year_published <= 1993.50
|   |   |   |   |   |--- Miniatures <= 0.50
|   |   |   |   |   |   |--- value: [6.16]
|   |   |   |   |   |--- Miniatures >  0.50
|   |   |   |   |   |   |--- value: [6.73]
|   |   |   |   |--- year_published >  1993.50
|   |   |   |   |   |--- Hex-and-Counter <= 0.50
|   |   |   |   |   |   |--- value: [6.47]
|   |   |   |   [...]<jupyter_text>RandomForestRegressor  Optimise n_estimators<jupyter_code>scores = []

for n in range(10, 201, 10):
    rf = RandomForestRegressor(n_estimators=n, random_state=1)
    rf.fit(X_train, y_train)

    y_pred = rf.predict(X_val)
    error = rmse(y_val, y_pred)
    
    scores.append((n, error))
df_scores = pd.DataFrame(scores, columns=['n_estimators', 'error'])
plt.plot(df_scores.n_estimators, df_scores.error)<jupyter_output><empty_output><jupyter_text>Optimise depth<jupyter_code>scores = []
for d in [5, 10, 15]:
    for n in range(10, 201, 10):
        rf = RandomForestRegressor(n_estimators=n, max_depth=d, random_state=1)
        rf.fit(X_train, y_train)

        y_pred = rf.predict(X_val)
        error = rmse(y_val, y_pred)
    
        scores.append((d, n, error))
columns = ['max_depth', 'n_estimators', 'error']
df_scores = pd.DataFrame(scores, columns=columns)
for d in [5, 10, 15]:
    df_subset = df_scores[df_scores.max_depth == d]
    
    plt.plot(df_subset.n_estimators, df_subset.error,
             label='max_depth=%d' % d)

plt.legend()
max_depth = 15<jupyter_output><empty_output><jupyter_text>Optimise min_samples_leaf<jupyter_code>scores = []

for s in [1, 3, 5, 10, 50]:
    for n in range(10, 201, 10):
        rf = RandomForestRegressor(n_estimators=n,
                                    max_depth=max_depth,
                                    min_samples_leaf=s,
                                    random_state=1)
        rf.fit(X_train, y_train)

        y_pred = rf.predict(X_val)
        error = rmse(y_val, y_pred)
    
        scores.append((s, n, error))
columns = ['min_samples_leaf', 'n_estimators', 'error']
df_scores = pd.DataFrame(scores, columns=columns)
colors = ['black', 'blue', 'orange', 'red', 'pink']
values = [1, 3, 5, 10, 50]

for s, col in zip(values, colors):
    df_subset = df_scores[df_scores.min_samples_leaf == s]
    
    plt.plot(df_subset.n_estimators, df_subset.error,
             color=col,
             label='min_samples_leaf=%d' % s)

plt.legend()
min_samples_leaf = 1<jupyter_output><empty_output><jupyter_text>Final RandomForestRegressor model<jupyter_code>rf = RandomForestRegressor(n_estimators=125,
                            max_depth=max_depth,
                            min_samples_leaf=min_samples_leaf,
                            random_state=1)

rf.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Validating the model using df_full train v df_test<jupyter_code>dv = DictVectorizer(sparse=False)

full_train_dict = df_full_train[game_col].to_dict(orient='records')
X_full_train = dv.fit_transform(full_train_dict)

test_dict = df_test[game_col].to_dict(orient='records')
X_test = dv.transform(test_dict)<jupyter_output><empty_output><jupyter_text>Testing DecisionTreeRegressor<jupyter_code>dt.fit(X_full_train, y_full_train)
    
y_pred = dt.predict(X_val)
error  = rmse(y_test, y_pred)
        
error
sns.histplot(y_pred, color='red', alpha=0.5, bins=50)
sns.histplot(y_test, color='blue', alpha=0.5, bins=50)<jupyter_output><empty_output><jupyter_text>Testing RandomForestRegressor<jupyter_code>rf.fit(X_full_train, y_full_train)

y_pred = rf.predict(X_test)
error  = rmse(y_test, y_pred)
error
sns.histplot(y_pred, color='red', alpha=0.5, bins=50)
sns.histplot(y_test, color='blue', alpha=0.5, bins=50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1404877085595814, 'fraction_numerical': 0.029815381577648337, 'mean_word_length': 3.8399426660296228, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 106, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '230195', 'license': 'CC0-1.0', 'n_tokens_mistral': 4171, 'n_tokens_neox': 3860, 'n_words': 603, 'path': 'mlzoomcamp/wk07_midterm_project/Notebook.ipynb', 'repo_name': 'jazwilson/workbook', 'size': '10129'}