<jupyter_start><jupyter_text><jupyter_code>!pip install  eli5

import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score

import eli5
from eli5.sklearn import PermutationImportance

from ast import literal_eval
from tqdm import tqdm_notebook

cd "/content/drive/My Drive/Colab Notebooks"
cd dw_matrix/
df = pd.read_csv('data/men_shoes.csv', low_memory=False)
df.columns
def run_model(feats, model = DecisionTreeRegressor(max_depth=5)):
  X = df[ feats ].values
  y = df['prices_amountmin'].values

  scores = cross_val_score(model,X, y, scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)
# df['brand_cat'] = df['brand'].factorize()[0]
# future engineering
df['brand_cat'] = df['brand'].map(lambda x: str(x).lower()).factorize()[0]
run_model(['brand_cat'])<jupyter_output><empty_output><jupyter_text>usprawniamy jakość modelu po zsatosowaniu - Random Forest<jupyter_code>model = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
run_model(['brand_cat'], model)<jupyter_output><empty_output><jupyter_text>Feature Engineering -  to lepszego poznania danych - operacje na danych- wyżej żeby brand było przrównanne wszytskie z małej litery<jupyter_code>df.head()
df.features.head().values
test = {'key': 'value'}
test['key']

str(test)
str_dict = '[{"key":"Gender","value":["Men"]},{"key":"Shoe Size","value":["M"]},{"key":"Shoe Category","value":["Men\'s Shoes"]},{"key":"Color","value":["Multicolor"]},{"key":"Manufacturer Part Number","value":["8190-W-NAVY-7.5"]},{"key":"Brand","value":["Josmo"]}]'
#literal_eval(str_dict) [0]['key']
#literal_eval(str_dict) [0]['value']
literal_eval(str_dict) [0]['value'][0]
#def parse_features(x):
 # if str(x) == 'nan': return [] # jeżeli wartość jest pusta to pusta lista
 # return literal_eval(x)
#replace('\\", '"'))
#  df['features_parsed'] = df['features'].map(parse_features)
 def parse_features(x):
  output_dict = {}
  if str(x) == 'nan': return output_dict
  features = literal_eval(x.replace('\\"', '"'))
  for item in features:
    key = item['key'].lower().strip()
    value = item['value'][0].lower().strip()
    output_dict[key] = value
  return output_dict
df['features_parsed'] = df['features'].map(parse_features)
df['features_parsed'].head().values
keys = set()

df['features_parsed'].map( lambda x: keys.update(x.keys()))

len(keys)

def get_name_feat(key):
  return 'feat_' + key
  
for key in tqdm_notebook(keys):
  #print(key)
  #break
  df[get_name_feat(key)] = df.features_parsed.map(lambda feats:feats[key]  if key in feats else np.nan)
df.columns
keys_stat = {}
for key in keys:
  keys_stat[key] = df[False == df[get_name_feat(key)].isnull()].shape[0] / df.shape[0] * 100
{k:v for k,v in keys_stat.items() if v > 30}
#output tylko gender występuje w 50% parametrów
#normalizacja jednostek wagi 
def normalize_to_gram(line):
    if str(line) == 'nan' : return -1
    

    value, unit = line.lower().split(' ')
    value = float(value)

    if unit == 'lbs' or unit == 'lb' or unit == 'pounds':
        value *= 453.592
    elif 'kg' == unit:
        value *= 1000
    elif 'ounces' == unit:
        value *= 28.3495
    elif 'g' == unit:
        pass
    else:
        print("ERROR: ", unit)
    return value

df['weight_norm'] = df['weight'].map(normalize_to_gram)
df['feat_brand_cat'] = df['feat_brand'].factorize()[0]
df['feat_color_cat'] = df['feat_color'].factorize()[0]
df['feat_gender_cat'] = df['feat_gender'].factorize()[0]
df['feat_manufacturer_cat'] = df['feat_manufacturer'].factorize()[0]
df['feat_material_cat'] = df['feat_material'].factorize()[0]

df['feat_sport_cat'] = df['feat_sport'].factorize()[0]
df['feat_size_cat'] = df['feat_size'].factorize()[0]
df['feat_weight_norm_cat'] = df['feat_weight'].factorize()[0]

#wersja z usprawnionym normlaizacją dla wszystkich _cat
for key in keys:
  df[get_name_feat(key) + '_cat'] = df[get_name_feat(key)].factorize()[0]
run_model(['feat_brand_cat'])
df['brand'] = df['brand'].map(lambda x: str(x).lower())
df[df.brand == df.feat_brand].shape
feats = ['']
model = RandomForestRegressor(max_depth=5, n_estimators=100)
run_model(['brand_cat'], model)
feats_cat = [x for x in df.columns if 'cat' in x]
feats_cat
feats = ['brand_cat','feat_brand_cat','feat_gender_cat','feat_material_cat','feat_size_cat', 'feat_weight_norm_cat']
#feats += feats_cat
#feats = list(set(feats)) # unikalna lista
model = RandomForestRegressor(max_depth=5, n_estimators=100)

result = run_model(feats, model)
X = df[feats].values
y = df['prices_amountmin'].values

m = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
m.fit(X,y)

print(result)
perm = PermutationImportance(m,random_state=1).fit(X,y);
eli5.show_weights(perm, feature_names=feats)<jupyter_output>(-57.00357565753129, 4.23338437860591)<jupyter_text>outputWeight	Feature0.2578 ± 0.0112	brand_cat0.1058 ± 0.0094	feat_material_cat0.0423 ± 0.0043	feat_gender_cat0.0213 ± 0.0011	feat_brand_cat0.0077 ± 0.0004	feat_color_cat - usuwamy tą cechę jako taką która nie wpływa na wynik0.0002 ± 0.0000	feat_manufacturer_cat - usuwamy tą cechę jako taką która nie wpływa na wynik 0.2595 ± 0.0080	brand_cat - widać że Brand znacząco wpływa na wynik - dlatego trzeba bliżej mu się przyjrzeć.<jupyter_code>df['brand'].value_counts(normalize=True)
df[df['brand'] == 'nike'].features_parsed.head().values
df[df['brand'] == 'nike'].features_parsed.sample(5).values
df['feat_age group'].value_counts()
!git add matrix_one/day5.ipynb

!git commit -m "DAY 5 - Read Men's Shoe Prices dataset from data.world"

!git config --global user.email "<EMAIL>"
!git config --global user.name "<NAME>"

!git commit -m "DAY 5 - Read Men's Shoe Prices dataset from data.world"
!git push -u origin master
!git pull origin master 
!git add  matrix_one/day5.ipynb 
!git commit -m "Read Men's Shoe Prices dataset from data.world"
!git push origin master<jupyter_output>From https://github.com/miklaszewskigrzegorz/dw_matrix
 * branch            master     -> FETCH_HEAD
error: Your local changes to the following files would be overwritten by merge:
	matrix_one/day4.ipynb
Please commit your changes or stash them before you merge.
Aborting
[master 4e5c426] Read Men's Shoe Prices dataset from data.world
 1 file changed, 1 insertion(+), 1 deletion(-)
To https://github.com/miklaszewskigrzegorz/dw_matrix.git
 ! [rejected]        master -> master (non-fast-forward)
error: failed to push some refs to 'https://906bab3a1f3626bf82055c14725d558af31c90e7@github.com/miklaszewskigrzegorz/dw_matrix.git'
hint: Updates were rejected because the tip of your current branch is behind
hint: its remote counterpart. Integrate the remote changes (e.g.
hint: 'git pull ...') before pushing again.
hint: See the 'Note about fast-forwards' in 'git push --help' for details.
{'fraction_non_alphanumeric': 0.1520797227036395, 'fraction_numerical': 0.030906990179087233, 'mean_word_length': 5.7363813229571985, 'pattern_counts': {'":': 12, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '133088', 'license': 'MIT', 'n_tokens_mistral': 2675, 'n_tokens_neox': 2536, 'n_words': 575, 'path': 'matrix_one/day5.ipynb', 'repo_name': 'miklaszewskigrzegorz/dw_matrix', 'size': '6962'}