<jupyter_start><jupyter_text>Product recommender system<jupyter_code>#import packages 
import sklearn
import pandas as pd
import numpy as np
import os<jupyter_output><empty_output><jupyter_text>Get the data<jupyter_code>from sklearn.datasets import load_wine
data1 = load_wine(as_frame = True)
#Load dataset
DIRECTORY_WHERE_THIS_FILE_IS = os.path.dirname(os.path.abspath("Product_recommender.md"))
DATA_PATH = os.path.join(DIRECTORY_WHERE_THIS_FILE_IS, "data/kaggle_wine2.csv")
df1 = pd.read_csv(DATA_PATH)
df1['description'].head(2)
#extract the wine year
import re
df1['year'] = 0
count = 0
for el in df1['title']: 
    res = [int(i) for i in el.split() if i.isdigit()]
    if len(res) == 1:
        df1.loc[count,'year'] = res[0]
    elif len(res) == 2 and res[0] > 1900:
        df1.loc[count,'year'] = res[0]
    elif len(res) == 2 and res[1] > 1900:
         df1.loc[count,'year'] = res[1]
    count += 1
#onehotencode the country qnd yeqr
categorical_columns = ['country', 'variety']
for column in categorical_columns:
    tempdf = pd.get_dummies(df1[column], prefix=column)
    df1 = pd.merge(
        left=df1,
        right=tempdf,
        left_index=True,
        right_index=True,
    )
    df1 = df1.drop(columns=column)
#drop the other non numericql collumns
df2 = df1.drop(['Unnamed: 0','designation','description','province','region_1','region_2','taster_name','taster_twitter_handle','winery'], axis = 1)
#Take the most represented collumns
for col in df2.columns:
    if col not in ('points', 'price','title'):
        if sum(df2[col]) < 5000 :
            df2 = df2.drop(columns= col)
df2.reset_index(inplace = True)
df_title = df2.loc[:,['index','title']]
df2 = df2.drop('title', axis = 1)
print(df_title.head(5))
df2.head(5)
#null
#drop for now
df2 = df2.dropna()
from sklearn.model_selection import train_test_split
y = df2["points"]
X = df2.loc[:,df2.columns != "points"]
#scale
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler().fit(X)
X = scaler.transform(X)
X = pd.DataFrame(X)

#remove correlated features
def get_correlation(data, threshold):
    corr_col = set()
    corrmat = data.corr()
    for i in range(len(corrmat.columns)):
        for j in range(i):
            if abs(corrmat.iloc[i, j]) > threshold:
                colname = corrmat.columns[i]
                corr_col.add(colname)
    return corr_col

corr_features = get_correlation(X, 0.70)
print('correlated features: ', len(set(corr_features)) )
print(corr_features)

#X = X.drop(labels=corr_features, axis = 1)


#train test
X_train, X_test, y_train, y_test = train_test_split(
   X, y, test_size=0.30, random_state=42)<jupyter_output><empty_output><jupyter_text>Baseline<jupyter_code>average = sum(y_train) / len(y_train)
arr_avg = []
for el in y_test:
    arr_avg.append(average)
print('Mean absolute error: {}'.format(mean_absolute_error(y_test,arr_avg)))<jupyter_output>Mean absolute error: 2.4913350809477026<jupyter_text>Random forest regressor<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestRegressor 
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
grid = {
    "max_depth": [35,40], 
    "min_samples_leaf": [2,3],
    "min_samples_split": [4,5,8,10]
}

"Sklearn"
"-----------------------"
grid_search = GridSearchCV(RandomForestRegressor(), param_grid = grid)
grid_search.fit(X_train, y_train)
optimal_model = grid_search.best_estimator_
"-----------------------"

print("Fine Tuned Model: {0}".format(optimal_model))
model = RandomForestRegressor(max_depth = optimal_model.max_depth, min_samples_leaf = optimal_model.min_samples_leaf, min_samples_split = optimal_model.min_samples_split)
model.fit(X_train,y_train)
adjusted_pred = model.predict(X_test)
print('Mean absolute error: {}'.format(mean_absolute_error(y_test,adjusted_pred)))<jupyter_output>Mean absolute error: 1.8147790713983896
{'fraction_non_alphanumeric': 0.14041794087665646, 'fraction_numerical': 0.029561671763506627, 'mean_word_length': 4.806213017751479, 'pattern_counts': {'":': 3, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '132203', 'license': 'MIT', 'n_tokens_mistral': 1495, 'n_tokens_neox': 1400, 'n_words': 288, 'path': 'Product_recommender.ipynb', 'repo_name': 'Pierre-mulliez1/WineMe', 'size': '3924'}