<jupyter_start><jupyter_text><jupyter_code>!pip install eli5
import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score

from ast import literal_eval
from tqdm import tqdm_notebook

import eli5
from eli5.sklearn import PermutationImportance
cd "/content/drive/My Drive/Colab Notebooks/dw_matrix"
ls data

df = pd.read_csv('data/men_shoes.csv', low_memory=False)
def run_model (feats, model = DecisionTreeRegressor(max_depth=5)):
  x = df[ feats].values
  y = df['prices_amountmin'].values

  scores = cross_val_score(model, x, y, scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)
df['brand_cat'] = df['brand'].map(lambda x: str(x).lower()).factorize()[0]
run_model(['brand_cat'])
model = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
run_model(['brand_cat'], model)

df.features.head().values
test = {'key':'value'}
test['key']

str(test)
str_dict = '[{"key":"Gender","value":["Men"]},{"key":"Shoe Size","value":["M"]},{"key":"Shoe Category","value":["Men\'s Shoes"]},{"key":"Color","value":["Multicolor"]},{"key":"Manufacturer Part Number","value":["8190-W-NAVY-7.5"]},{"key":"Brand","value":["Josmo"]}]'

literal_eval(str_dict)[0]['value']
def parse_features(x):
  output_dict = {}
  if str(x) == 'nan' : return output_dict

  features =  literal_eval(x.replace('\\"','"'))
  for item in features: 
    key = item['key'].lower().strip()
    value = item ['value'][0].lower().strip()

    output_dict[key] = value

  return output_dict

df['features_parsed'] = df['features'].map(parse_features)
keys = set()
df['features_parsed'].map( lambda x: keys.update(x.keys()) )
len(keys)
def get_name_feat(key):
  return 'feat_'+ key

for key in tqdm_notebook(keys):
  df[get_name_feat(key)] = df.features_parsed.map(lambda feats: feats[key] if key in feats else np.nan)
df.columns
keys_stat = {}
for key in keys:
  keys_stat[key] = df[ False== df[get_name_feat(key)].isnull() ].shape[0] / df.shape[0] * 100
{k:v for k,v in keys_stat.items() if v>30}
df['feat_brand_cat'] = df['feat_brand'].factorize()[0]
df['feat_color_cat'] = df['feat_color'].factorize()[0]
df['feat_gender_cat'] = df['feat_gender'].factorize()[0]
df['feat_manufacturer part number_cat'] = df['feat_manufacturer part number'].factorize()[0]
df['feat_material_cat'] = df['feat_material'].factorize()[0]

df['feat_sport_cat'] = df['feat_sport'].factorize()[0]
df['feat_style_cat'] = df['feat_style'].factorize()[0]

for key in keys:
  df[get_name_feat(key) + '_cat'] = df[get_name_feat(key)].factorize()[0]
  
df ['brand'] = df ['brand'].map(lambda x: str(x).lower() )
df[ df.brand == df.feat_brand ].shape
feats = ['']
model = RandomForestRegressor(max_depth=5, n_estimators=100)
run_model('brand_cat', model)

feats_cat = [x for x in df.columns if 'cat' in x]
feats_cat
feats = ['brand_cat','feat_metal type_cat','feat_brand_cat', 'feat_gender_cat', 'feat_material_cat', 'feat_style_cat', 'feat_sport_cat', 'feat_shape_cat' ]

model = RandomForestRegressor(max_depth=5, n_estimators=100)
result = run_model(feats, model)
x = df[ feats ].values
y = df['prices_amountmin'].values
m = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
m.fit(x,y)

print(result)
perm = PermutationImportance(m, random_state=1).fit(x,y);
eli5.show_weights(perm, feature_names=feats)
df[ df['brand'] == 'nike'].features_parsed.sample(5).values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1800561797752809, 'fraction_numerical': 0.013764044943820225, 'mean_word_length': 7.262180974477959, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '287227', 'license': 'MIT', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1310, 'n_words': 249, 'path': 'day5.ipynb', 'repo_name': 'wiktoriaslomiany/dw_matrix', 'size': '3560'}