<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score
cd "/content/drive/My Drive/Colab Notebooks/dw_matrix"
df=pd.read_csv("data/men_shoes.csv",low_memory=False)
df.shape<jupyter_output><empty_output><jupyter_text><jupyter_code>df.columns
mean_price=np.mean(df['prices_amountmin'])
mean_price
y_true=df['prices_amountmin']
y_true.shape[0]
y_pred=[mean_price]*y_true.shape[0]
mean_absolute_error(y_true,y_pred)
np.log1p(df['prices_amountmin']).hist(bins=100)
y_true=df['prices_amountmin']
y_pred=[np.median(y_true)]*y_true.shape[0]
mean_absolute_error(y_true,y_pred)
y_true = df['prices_amountmin']
price_log_mean = np.expm1(np.mean(np.log1p(y_true)))
y_pred = [price_log_mean] * y_true.shape[0]

mean_absolute_error(y_true, y_pred)
np.exp(np.mean(np.log1p(y_true)))-1

df['brand_cat']=df['brand'].factorize()[0]
feats=['brand_cat']
X=df[feats].values
y=df['prices_amountmin'].values

model =DecisionTreeRegressor(max_depth=5)
scores=cross_val_score(model,X,y,scoring='neg_mean_absolute_error')
np.mean(scores), np.std(scores)

def run_model(feats):
  X=df[feats].values
  y=df['prices_amountmin'].values

  model =DecisionTreeRegressor(max_depth=5)
  scores=cross_val_score(model,X,y,scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)
run_model(['brand_cat'])
df['reviews_cat']=df['reviews'].factorize()[0]
run_model(['reviews_cat'])
run_model(['brand_cat','reviews_cat'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15408608858390518, 'fraction_numerical': 0.009981285090455396, 'mean_word_length': 11.73015873015873, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '337944', 'license': 'MIT', 'n_tokens_mistral': 664, 'n_tokens_neox': 647, 'n_words': 69, 'path': 'day4.ipynb', 'repo_name': 'DominikJarczyk/dw_matrix', 'size': '1603'}