<jupyter_start><jupyter_text>Read data from CSV and check what is inside<jupyter_code>df = pd.read_csv("data/shoes_prices.csv", low_memory=False)
df.shape
df.columns<jupyter_output><empty_output><jupyter_text>Basics stats and prediction<jupyter_code>mean_price = np.mean(df["prices_amountmin"])
mean_price
   y_true = df["prices_amountmin"]
   y_true.shape[0]<jupyter_output><empty_output><jupyter_text>MEAN<jupyter_code>y_pred = [mean_price] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)
df.prices_amountmin.hist(bins=100);<jupyter_output><empty_output><jupyter_text>To avoid log of 0 use `np.log1p == np.log(x+1)`<jupyter_code>np.log1p(df.prices_amountmin+1).hist(bins=100);<jupyter_output><empty_output><jupyter_text>MEDIAN<jupyter_code>price_median = np.median(df["prices_amountmin"])
y_pred = [price_median] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)<jupyter_output><empty_output><jupyter_text>LOG MEAN<jupyter_code>price_log_mean = np.expm1( np.mean( np.log1p(y_true) ) ) 
 y_pred = [price_log_mean] * y_true.shape[0]  
mean_absolute_error(y_true, y_pred)<jupyter_output><empty_output><jupyter_text>Decision tree<jupyter_code>df.columns
df.brand.value_counts()<jupyter_output><empty_output><jupyter_text>String data must me normalized to Integer<jupyter_code>df['brand_cat'] = df.brand.factorize()[0]
feats = ['brand_cat']
X = df[ feats ].values
y = df['prices_amountmin'].values

model = DecisionTreeRegressor(max_depth=5)

score = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
np.mean(score), np.std(score)<jupyter_output><empty_output><jupyter_text>Define funcion<jupyter_code>def run_model(feats):
  X = df[ feats ].values
  y = df['prices_amountmin'].values

  model = DecisionTreeRegressor(max_depth=5)

  score = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
  return np.mean(score), np.std(score)
run_model(['brand_cat'])
df['colors_cat'] = df.colors.factorize()[0]
run_model(['colors_cat'])
run_model(['brand_cat', 'colors_cat'])
ls
!git add matrix_1/day4.ipynb
!git commit -m "First ML model"
!git push -u origin master<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14150943396226415, 'fraction_numerical': 0.010849056603773584, 'mean_word_length': 9.5, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '646802', 'license': 'MIT', 'n_tokens_mistral': 873, 'n_tokens_neox': 849, 'n_words': 113, 'path': 'matrix_1/day4.ipynb', 'repo_name': 'michkowalczuk/dw_matrix', 'size': '2120'}