<jupyter_start><jupyter_text><jupyter_code>!pip install --upgrade tables
!pip install eli5
cd "/content/drive/My Drive/Colab Notebooks/Matrix/dw_matrix_2"
ls
import pandas as pd
import numpy as np

from sklearn.dummy import DummyRegressor
from sklearn.tree import DecisionTreeRegressor

from sklearn.metrics import mean_absolute_error as mae
from sklearn.model_selection import cross_val_score

import eli5
from eli5.sklearn import PermutationImportance<jupyter_output><empty_output><jupyter_text>data import<jupyter_code>df = pd.read_hdf('data/car.h5')
df.shape
df.columns<jupyter_output><empty_output><jupyter_text>Dummy Model<jupyter_code>df.select_dtypes(np.number).columns
feats = ['car_id']
X = df[feats].values
y= df['price_value'].values


model = DummyRegressor()
model.fit(X,y)
y_pred = model.predict(X)
mae(y,y_pred)
ytest = model.predict([121]); ytest
ytest
df['price_value'].values.mean()
df = df[df['price_currency']!= 'EUR']
feats = ['car_id']
X = df[feats].values
y= df['price_value'].values


model = DummyRegressor()
model.fit(X,y)
y_pred = model.predict(X)
mae(y,y_pred)
for feat in df.columns:
  if isinstance(df[feat][0], list): continue
  factorized_values = df[feat].factorize()[0]
  if '__cat' in feat: continue
  df[feat+'__cat'] = factorized_values

cat_feats = [x for x in df.columns if '__cat' in x]
len(cat_feats)
cat_feats = [x for x in cat_feats if 'price' not in x]
len(cat_feats)
X = df[cat_feats].values
y =df['price_value'].values

model = DecisionTreeRegressor(max_depth=5)
scores = cross_val_score(model, X, y, cv = 3, scoring = 'neg_mean_absolute_error')
np.mean(scores)
m = DecisionTreeRegressor(max_depth=5)
m.fit(X, y)
imp = PermutationImportance(m, random_state=0).fit(X,y)
eli5.show_weights(imp, feature_names = cat_feats)
!git init
!git add day3_model.ipynb
ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1292257360959651, 'fraction_numerical': 0.008724100327153763, 'mean_word_length': 6.8418803418803416, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '457118', 'license': 'MIT', 'n_tokens_mistral': 729, 'n_tokens_neox': 706, 'n_words': 152, 'path': 'day3_model.ipynb', 'repo_name': 'dtandev/dw_matrix_2', 'size': '1834'}