<jupyter_start><jupyter_text>Wczytywanie danych<jupyter_code>cd '/content/drive/My Drive/Colab Notebooks/dw_matrix/matrix_two/dw_matrix_car'
df = pd.read_hdf('data/car.h5')
df.shape
df.columns<jupyter_output><empty_output><jupyter_text>Dummy model<jupyter_code>df.select_dtypes(np.number).columns
feats = ['car_id']
X= df [feats].values
y= df ['price_value'].values

model = DummyRegressor()
model.fit (X,y)
y_pred = model.predict(X)

mae(y, y_pred)
[x for x in df.columns if 'price' in x]
df['price_currency'].value_counts(normalize=True)*100
df =df [df['price_currency'] != 'EUR']
df.shape<jupyter_output><empty_output><jupyter_text>Features<jupyter_code>df.head()
df['param_color'].factorize()[0]
for feat in df.columns:
  print(feat)
SUFFIX_CAT = '__cat'
for feat in df.columns:
  if isinstance (df[feat][0],list): continue

  factorized_values = df[feat].factorize()[0]
  if SUFFIX_CAT in feat:
    df[feat] = factorized_values
  else:
    df[feat + SUFFIX_CAT] = factorized_values
cat_feats = [x for x in df.columns if SUFFIX_CAT in x]
cat_feats = [x for x in cat_feats if 'price' not in x]
len(cat_feats)
X = df[cat_feats].values
y = df['price_value'].values

model = DecisionTreeRegressor(max_depth = 5)
scores = cross_val_score(model, X, y, cv = 3, scoring = 'neg_mean_absolute_error')

np.mean(scores)<jupyter_output><empty_output><jupyter_text>sprawdzenie, ktore cechy sa ciekawsze<jupyter_code>m = DecisionTreeRegressor(max_depth=5)
m.fit(X, y)

imp = PermutationImportance(m, random_state=0).fit(X, y)
eli5.show_weights(imp, feature_names = cat_feats)
df.groupby('param_napęd__cat')['price_value'].mean()
df.groupby('param_napęd__cat')['price_value'].agg(np.mean).plot(kind='bar')
(
  df
  .groupby('param_napęd__cat')['price_value']
  .agg(np.mean)
  .sort_values(ascending=False)
  .head(50)
).plot(kind='bar',figsize=(15,5))
def group_and_barplot(feat_groupby, feat_agg='price_value', agg_funcs=[np.mean, np.median, np.size], feat_sort='size', subplots = True):
  return(
  df
  .groupby('param_napęd__cat')[feat_agg]
  .agg(agg_funcs)
  .sort_values(by=feat_sort, ascending=False)

).plot(kind='bar',figsize=(15,5), subplots=subplots)
group_and_barplot('param_napęd__cat');
ls
cd '/content/drive/My Drive/Colab Notebooks/dw_matrix'
ls
cd 'matrix_three'
ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1529105125977411, 'fraction_numerical': 0.008688097306689836, 'mean_word_length': 7.137809187279152, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '579809', 'license': 'MIT', 'n_tokens_mistral': 966, 'n_tokens_neox': 936, 'n_words': 156, 'path': 'matrix_three/matrix_2_3_simple_model.ipynb', 'repo_name': 'inkiga/dw_matrix', 'size': '2302'}