<jupyter_start><jupyter_text>Loading the data<jupyter_code>df = pd.read_hdf('data/car.h5')
df.shape<jupyter_output><empty_output><jupyter_text>Dummy model<jupyter_code>df.select_dtypes(np.number).columns
feats = ['car_id']
X = df[feats].values
y = df['price_value'].values

model = DummyRegressor()
model.fit(X,y)
y_pred = model.predict(X)

mae (y, y_pred)
[x for x in df.columns if 'price' in x]
df['price_currency'].value_counts(normalize=True) * 100
df = df[df['price_currency'] != 'EUR']
df.shape
df.columns<jupyter_output><empty_output><jupyter_text>Features<jupyter_code>SUFFIX_CAT = '__cat'

for feat in df.columns:
  if isinstance (df[feat][0], list): continue
  factorized_values = df[feat].factorize()[0]

  if SUFFIX_CAT in feat:
    df[feat] = factorized_values
  else:
    df[feat + SUFFIX_CAT] = factorized_values
cat_feats = [x for x in df.columns if SUFFIX_CAT in x]
cat_feats = [x for x in cat_feats if 'price' not in x]
len(cat_feats)
X = df[cat_feats].values
y = df['price_value'].values

model = DecisionTreeRegressor(max_depth=5)
scores = cross_val_score(model, X, y, cv=3, scoring='neg_mean_absolute_error')

np.mean(scores)

X
m = DecisionTreeRegressor(max_depth=5)
m.fit(X,y)

imp =PermutationImportance(m, random_state=0).fit(X,y)
feature_importance = eli5.show_weights(imp, feature_names=cat_feats)
feature_importance
imp_columns = ['param_napęd__cat',
           'param_faktura-vat__cat',
           'param_stan__cat',
           'param_rok-produkcji__cat',
           'param_moc__cat',
           'feature_kamera-cofania__cat',
           'param_skrzynia-biegów__cat',
           'param_marka-pojazdu__cat']
cat_feats = [x for x in cat_feats if x in imp_columns]
len(cat_feats)
X = df[cat_feats].values
y = df['price_value'].values

model = DecisionTreeRegressor(max_depth=7)
scores = cross_val_score(model, X, y, cv=3, scoring='neg_mean_absolute_error')

np.mean(scores)
(
    df
      .groupby('param_napęd')['price_value']
      .agg([np.median, np.size])
      .sort_values(by='size', ascending=True)
      .head(50)
).plot(kind='pie', figsize=(14,10), subplots=True);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13896713615023473, 'fraction_numerical': 0.00892018779342723, 'mean_word_length': 5.158959537572255, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '425423', 'license': 'MIT', 'n_tokens_mistral': 866, 'n_tokens_neox': 839, 'n_words': 147, 'path': 'd3-model.ipynb', 'repo_name': 'Edith1100/dw_matrix_car', 'size': '2130'}