<jupyter_start><jupyter_text>Main category in purchases view<jupyter_code>def main_cat_purchase(session_features):
    carousel = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_CAROUSEL'].copy(), 'ocarproducts')
    carousel['prod_counter'] = carousel.product_list.apply(cat_counter)
    session_carousel = carousel.groupby('sid').prod_counter.agg(merge_counters).reset_index()
    
    lp = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_LP'].copy(), 'products')
    lp['prod_counter'] = lp.product_list.apply(cat_counter)
    session_lp = lp.groupby('sid').prod_counter.agg(merge_counters).reset_index()
    
    lr = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_LR'].copy(), 'oproducts')
    lr['prod_counter'] = lr.product_list.apply(cat_counter)
    session_lr = lr.groupby('sid').prod_counter.agg(merge_counters).reset_index()
    
    session_categories = pd.merge(pd.merge(session_carousel, session_lp, on='sid', how='left'), session_lr, on='sid', how='left')
    session_categories['prod_counters'] = list(zip(session_categories.prod_counter_car, session_categories.prod_counter_lp, session_categories.prod_counter_lr))
    
    def merge_xyz(row):
        counters = []
        for i in range(3):
            if pd.notnull(row[i]):
                counters.append(row[i])
        merged = merge_counters(counters)
        evaluate = Counter(merged)
        return evaluate.most_common(1)[0][0]
    session_categories['top_cat'] = session_categories.prod_counters.apply(merge_xyz)
    top_cat_sessions = session_categories[['sid', 'top_cat']]
    top_cat_sessions.columns = ['sid', 'MAIN_CATEGORY_PURCHASED_VIEW']
    session_features = pd.merge(session_features, top_cat_sessions, on='sid', how='left')
    return session_features
session_data = []

carousel = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_CAROUSEL'].copy(), 'ocarproducts')
carousel['prod_counter0'] = carousel.product_list.apply(cat_counter)
session_carousel = carousel.groupby('sid').prod_counter0.agg(merge_counters).reset_index()
session_data.append(session_carousel)

lp = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_LP'].copy(), 'products')
lp['prod_counter1'] = lp.product_list.apply(cat_counter)
session_lp = lp.groupby('sid').prod_counter1.agg(merge_counters).reset_index()
session_data.append(session_lp)

lr = convert_jsonproducts(train_tracking[train_tracking.type=='PURCHASE_PRODUCT_LR'].copy(), 'oproducts')
lr['prod_counter2'] = lr.product_list.apply(cat_counter)
session_lr = lr.groupby('sid').prod_counter2.agg(merge_counters).reset_index()
session_data.append(session_lr)

bkt = convert_jsonproducts(train_tracking[train_tracking.type=='ADD_TO_BASKET_CAROUSEL'].copy(), 'ocarproducts')
bkt['prod_counter3'] = bkt.product_list.apply(cat_counter)
session_bkt = bkt.groupby('sid').prod_counter3.agg(merge_counters).reset_index()
session_data.append(session_bkt)

len(carousel), len(lr), len(lp), len(bkt)
bkt = convert_jsonproducts(train_tracking[train_tracking.type=='ADD_TO_BASKET_LR'].copy(), 'oproducts')
bkt = bkt.drop(bkt[pd.isnull(bkt.oproducts)].index, axis=0)
bkt['prod_counter5'] = bkt.product_list.apply(cat_counter)
session_bkt = bkt.groupby('sid').prod_counter5.agg(merge_counters).reset_index()
session_data.append(session_bkt)

len(bkt)
session_categories = pd.merge(pd.merge(session_carousel, session_lp, on='sid', how='left'), session_lr, on='sid', how='left')
session_categories['prod_counters'] = list(zip(session_categories.prod_counter_car, session_categories.prod_counter_lp, session_categories.prod_counter_lr))
# session_categories.applymap(merge_xyz)
# session_categories
def merge_xyz(row):
    counters = []
    for i in range(3):
        if pd.notnull(row[i]):
            counters.append(row[i])
    merged = merge_counters(counters)
    evaluate = Counter(merged)
    return evaluate.most_common(1)[0][0]
session_categories['top_cat'] = session_categories.prod_counters.apply(merge_xyz)
top_cat_sessions = session_categories[['sid', 'top_cat']]
top_cat_sessions.columns = ['sid', 'MAIN_CATEGORY_PURCHASED_VIEW']
var = pd.merge(session_features, top_cat_sessions, on='sid', how='left')
len(var[pd.notnull(var.MAIN_CATEGORY_PURCHASED_VIEW)])
# top_cat_sessions
275/len(session_features)<jupyter_output><empty_output><jupyter_text>OCAR Products main category<jupyter_code>session_data = []

ocarprods = convert_jsonproducts(train_tracking[pd.notnull(train_tracking.ocarproducts)].copy(), 'ocarproducts')
ocarprods['prod_counter'] = ocarprods.product_list.apply(cat_counter)
session_ocar = ocarprods.groupby('sid').prod_counter.agg(merge_counters).reset_index()
len(session_ocar)
25484/len(session_features)<jupyter_output><empty_output><jupyter_text>Main watched product<jupyter_code>def watched_category(session):
    prods = fast_convert_jsonproducts(train_tracking[pd.notnull(train_tracking.products)].copy(), 'products')
    prods['prod_counter'] = prods.product_list.apply(partial(cat_counter, catmap=catmap))
    session_prods = prods.groupby('sid').prod_counter.agg(merge_counters).reset_index()
    
    def top_cat(x):
        evaluation = Counter(x)
        return evaluation.most_common(1)[0][0]
    session_prods['top_cat'] = session_prods.prod_counter.apply(top_cat)
    
    session_cat = session_prods[['sid', 'top_cat']].copy()
    session_cat.columns = ['sid', 'WATCHED_CATEGORY']
    session_features = pd.merge(session_features, session_cat, on='sid', how='left')
    
    return session_features

def watched_product(session_features, train_tracking, product):
    prods = fast_convert_jsonproducts(train_tracking[pd.notnull(train_tracking.products)].copy(), 'products')
    prods['prod_counter'] = prods.product_list.apply(prod_counter)
    session_prods = prods.groupby('sid').prod_counter.agg(merge_counters).reset_index()
    
    def top_cat(x):
        evaluation = Counter(x)
        return evaluation.most_common(1)[0][0]
    session_prods['top_cat'] = session_prods.prod_counter.apply(top_cat)
    
    session_cat = session_prods[['sid', 'top_cat']].copy()
    session_cat.columns = ['sid', 'WATCHED_PRODUCT']
    session_features = pd.merge(session_features, session_cat, on='sid', how='left')
    
    return session_features
len(train_tracking[pd.notnull(train_tracking.products)])

test_data = train_tracking[pd.notnull(train_tracking.products)].sample(10000).copy()

def test(f1, f2):
    return sum(test_data.products.apply(lambda val: f1(val) == f2(val)))
prog = re.compile("'sku':\ *'([a-zA-Z0-9\+\=\/]+)'")
test(lambda val: len(re.findall(prog, val)), lambda x: len(literal_eval(x)))/10000
prods = fast_convert_jsonproducts(train_tracking[pd.notnull(train_tracking.products)].copy(), 'products')
prods['prod_counter'] = prods.product_list.apply(partial(prod_counter))
session_prods = prods.groupby('sid').prod_counter.agg(merge_counters).reset_index()
len(session_prods)
len(session_prods)/len(session_features)
def top_cat(x):
    evaluation = Counter(x)
    return evaluation.most_common(1)[0][0]
session_prods['top_cat'] = session_prods.prod_counter.apply(top_cat)
session_cat = session_prods[['sid', 'top_cat']].copy()
session_cat.columns = ['sid', 'WATCHED_PRODUCT']
session_features = pd.merge(session_features, session_cat, on='sid', how='left')
train_features = watched_product(train_features, train_tracking, product)
test_features = watched_product(test_features, test_tracking, product)
train_features[['sid', 'WATCHED_PRODUCT']].to_feather(PROCESSED/'train_WP.feather')
test_features[['sid', 'WATCHED_PRODUCT']].to_feather(PROCESSED/'test_WP.feather')
len(train_features.WATCHED_PRODUCT.unique())/len(train_features)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14169779724333376, 'fraction_numerical': 0.0066984413242303234, 'mean_word_length': 7.402597402597403, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '505583', 'license': 'MIT', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2650, 'n_words': 303, 'path': 'notebooks/franco/03 - Categories Features.ipynb', 'repo_name': 'Robert-Alonso/DSG-2018-Finals', 'size': '7763'}