<jupyter_start><jupyter_text>Manufacturer<jupyter_code>df['manufacturer_cat'] =df.manufacturer.factorize()[0]
feats = ['manufacturer_cat', 'brand_cat']
run_model(feats)
model = RandomForestRegressor(max_depth=5, n_estimators=100, random_state=0)
run_model(['brand_cat'], model)
df.head()
def parse_features(x):
  output_dict = {}
  if str(x) == 'nan': return output_dict
  features =  literal_eval(x.replace('\\"', '"'))
  for item in features:
    key = item['key'].lower().strip()
    value = item['value'][0].lower().strip()
    output_dict[key] = value
  return output_dict

df['features_parsed'] = df['features'].map(parse_features)
keys = set()
# type(df['features_parsed'][0])
df['features_parsed'].map(lambda x: keys.update(x.keys()))
len(keys)
def get_name_feat(key):
  return 'feat_'+key

for key in tqdm_notebook(keys):
  df[get_name_feat(key)] = df.features_parsed.map(lambda feats: feats[key] if key in feats else np.nan)
keys_stat = {}
for key in keys:
  keys_stat[key] = df[False == df[get_name_feat(key)].isnull()].shape[0] / df.shape[0]*100
{k: v for k, v in keys_stat.items() if v> 30}
df['feat_brand_cat'] =df['feat_brand'].factorize()[0]
df['feat_color_cat'] =df['feat_color'].factorize()[0]
df['feat_gender_cat'] =df['feat_gender'].factorize()[0]
df['feat_manufacturer part number_cat'] =df['feat_manufacturer part number'].factorize()[0]
df['feat_material_cat'] =df['feat_material'].factorize()[0]
df['brand'] = df['brand'].map(lambda x : str(x).lower())
df[df.brand == df.feat_brand][['brand', 'feat_brand']].shape

feats = ['brand_cat', 'feat_color_cat', 'feat_gender_cat', 'feat_manufacturer part number_cat', 'feat_material_cat']
results = run_model(feats, model)
X = df[feats].values
y = df['prices_amountmin'].values
print
m = RandomForestRegressor(max_depth=5, n_estimators=100)
m.fit(X, y)
perm = PermutationImportance(m, random_state=1).fit(X, y)
eli5.show_weights(perm, feature_names=feats)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18442622950819673, 'fraction_numerical': 0.01331967213114754, 'mean_word_length': 7.6415929203539825, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '575005', 'license': 'MIT', 'n_tokens_mistral': 747, 'n_tokens_neox': 733, 'n_words': 129, 'path': 'part1/day4.ipynb', 'repo_name': 'jbanaszkiewicz/dw_matrix', 'size': '1952'}