<jupyter_start><jupyter_text>Domains<jupyter_code>domains = []
uh = train.user_history.values
for l in tqdm(uh):
    current = []
    for d in l:
        event_info = d['event_info']
        if isinstance(event_info, int):
            if domain[event_info] in domain_test_items and domain[event_info] != None:
                current.append(domain[event_info])
    domains.append(current)
    

domains_test = []
uh = test.user_history.values
for l in tqdm(uh):
    current = []
    for d in l:
        event_info = d['event_info']
        if isinstance(event_info, int):
            if domain[event_info] in domain_train_items and domain[event_info] != None:
                current.append(domain[event_info])
    domains_test.append(current)
    
domains_total = domains + domains_test
x = get_csr_matrix(domains_total).astype(np.float32)
clf_dm = ComplementNB(alpha=102)
clf_dm.fit(x[:len(train)], y_train_domain)
def parallelize_predict_proba(clf, x):

    def predict(clf, x_loc):
        p = clf.predict_proba(x_loc)
        proba = np.sort(p, axis=1)[:, :-11:-1]
        indices = p.argsort(axis=1)[:, :-11:-1]
        preds_10 = []
        for i in tqdm(indices):
            preds_10.append(list(clf.classes_[i]))
        return proba, preds_10
    
    chunks_locks = np.array_split(np.arange(x.shape[0], dtype=int), 60)
    
    with parallel_backend('loky', n_jobs=60):
        pool_result = Parallel(verbose=5)(delayed(predict)(clf, x[loc]) for loc in chunks_locks)
        
    proba = []
    for i in tqdm(pool_result):
        proba.append(i[0])
    proba = np.concatenate(tuple(proba), axis=0)
    
    preds = []
    for i in tqdm(pool_result):
        preds.extend(i[1])
        
    return proba, preds
proba_domain, preds_domain = parallelize_predict_proba(clf_dm, x)
# with open('./proba_domain_finaltest.pickle', 'wb') as handle:
#     pickle.dump(proba_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
# with open('./preds_domain_finaltest.pickle', 'wb') as handle:
#     pickle.dump(preds_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)

# with open('./proba_domain_finaltest.pickle', 'rb') as f:
#     proba_domain = pickle.load(f)

# with open('./preds_domain_finaltest.pickle', 'rb') as f:
#     preds_domain = pickle.load(f)
# with open('./proba_domain.pickle', 'wb') as handle:
#     pickle.dump(proba_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
# with open('./preds_domain.pickle', 'wb') as handle:
#     pickle.dump(preds_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)

# with open('./proba_domain.pickle', 'rb') as f:
#     proba_domain = pickle.load(f)

# with open('./preds_domain.pickle', 'rb') as f:
#     preds_domain = pickle.load(f)
accuracy_score(y_test_domain, [i[0] for i in preds_domain_test])<jupyter_output><empty_output><jupyter_text>Text Pega só o primeiro palpite do domínio. Acurácias: 1o palipite: 23%, 2o palipite: 6%, 3o: 3% (no conjunto teste)<jupyter_code>def get_queries(df):
    user_history = df.user_history.values
    queries = []
    for uh in tqdm(user_history):
        s = []
        for d in uh:
            if d['event_type'] == 'search':
                s.append(d['event_info'])
        queries.append(s)
    queries = [' '.join(list(set(i))).lower() for i in queries]
    return queries

queries = get_queries(train)

queries_test = get_queries(test)
def preprocess_text(text):
    text = re.sub(r'\d+', '', text)
    return text

cv = CountVectorizer(stop_words=stopwords, preprocessor=preprocess_text, max_df=0.2, min_df=0.0001, ngram_range=(1,2))

bag_of_words = cv.fit_transform(queries + queries_test)
clf = ComplementNB(alpha=100)
clf.fit(bag_of_words[:len(train)], y_train_domain)
def parallelize_predict_proba(clf, x):

    def predict(clf, x_loc):
        p = clf.predict_proba(x_loc)
        proba = np.sort(p, axis=1)[:, :-11:-1]
        indices = p.argsort(axis=1)[:, :-11:-1]
        preds_10 = []
        for i in tqdm(indices):
            preds_10.append(list(clf.classes_[i]))
        return proba, preds_10
    
    chunks_locks = np.array_split(np.arange(x.shape[0], dtype=int), 60)
    
    with parallel_backend('loky', n_jobs=60):
        pool_result = Parallel(verbose=5)(delayed(predict)(clf, x[loc]) for loc in chunks_locks)
        
    proba = []
    for i in tqdm(pool_result):
        proba.append(i[0])
    proba = np.concatenate(tuple(proba), axis=0)
    
    preds = []
    for i in tqdm(pool_result):
        preds.extend(i[1])
        
    return proba, preds
proba_text, preds_text = parallelize_predict_proba(clf, bag_of_words)
# proba_text, preds_text = parallelize_predict_proba(clf, bag_of_words[len(train):])
# with open('./proba_text_finaltest.pickle', 'wb') as handle:
#     pickle.dump(proba_text, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
# with open('./preds_text_finaltest.pickle', 'wb') as handle:
#     pickle.dump(preds_text, handle, protocol=pickle.HIGHEST_PROTOCOL)

# with open('./proba_text_finaltest.pickle', 'rb') as f:
#     proba_text = pickle.load(f)

# with open('./preds_text_finaltest.pickle', 'rb') as f:
#     preds_text = pickle.load(f)
# with open('./proba_text.pickle', 'wb') as handle:
#     pickle.dump(proba_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
# with open('./preds_text.pickle', 'wb') as handle:
#     pickle.dump(preds_domain, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
# with open('./proba_text.pickle', 'rb') as f:
#     proba_text = pickle.load(f)

# with open('./preds_text.pickle', 'rb') as f:
#     preds_text = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Ensemble<jupyter_code>def get_item_score(item, domain):
    try:
        index = domain_inverse_sorted[domain].index(item)
        return domain_inverse_sorted_score[domain][index]
    except:
        return 0

def generate_datapoint(v, y, domains_10, text_10, proba_domains_10, proba_text_10):
    candidates = np.unique(v + list(domain_inverse_sorted[domains_10[0]][:10]) + list(domain_inverse_sorted[domains_10[1]][:2]))


    nunique = len(np.unique(v))
    lenn = len(v)
    counts = counter(v)
    rec_score = get_recurrent_datapoint(v)
    prices = [0] if len(v) == 0 else [item_info[i][0] for i in v]
    maxx = np.max(prices)
    minn = np.min(prices)
    amp = maxx - minn
    
    try:
        unique_domains = get_unique_domains(v)
    except:
        unique_domains = 1

    datapoints = []
    for item in candidates:
        datapoints.append({
            'is_first': 1 if len(v) > 0 and item == v[0] else 0,
            'is_last': 1 if len(v) > 0 and item == v[-1] else 0,
            'count': counts.get(item, 0),
            'nunique': nunique,
            'max_price_viewed': maxx,
            'min_price_viewed': minn,
            'amp_price': amp,
            'lenn': lenn,
            'rec_score': rec_score.get(item, 0),
            'prob_domain': get_domain_score(domain[item], domains_10, proba_domains_10),
            'prob_text': get_domain_score(domain[item], text_10, proba_text_10),
            'prob_item': get_item_score(item, domain[item]),
            'unique_domains': unique_domains,
            'price': item_info[item][0],
            'condition': item_info[item][1],
            'target': 1 if y == item else 0
        })

    datapoints = pd.DataFrame().from_records(datapoints)
    return datapoints
    
datapoints = []
for i in tqdm(range(len(train[:100000]))):
    datapoints.append(generate_datapoint(views[i], y_train[i], preds_domain[i], preds_text[i], proba_domain[i], proba_text[i]))

datapoints = pd.concat(datapoints).reset_index(drop=True)
def dp_fe(df):
    df['perc'] = df['count']/df['lenn']
    df['prob_item_domain'] = df['prob_domain']*df['prob_item']
    df['prob_item_text'] = df['prob_text']*df['prob_item']
    return df

datapoints = dp_fe(datapoints)

datapoints = undersample(datapoints)
# datapoints.to_pickle('./datapoints.pickle')

# with open('./datapoints.pickle', 'rb') as f:
#     datapoints = pickle.load(f)
dp_x = datapoints.drop('target', axis=1)
dp_y = datapoints['target']

dp_x_train, dp_x_test, dp_y_train, dp_y_test = train_test_split(dp_x, dp_y, test_size=0.25, random_state=42, stratify=dp_y)
datapoints
lr = 0.12
lr_decay = 0.98


params = {
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'metric': 'binary_logloss',
    'num_leaves': 512,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.8,
    'lambda_l2': 1.2, #0.65,
    'lambda_l1': 1.2, #0.65,
    'bagging_freq': 5,
    'num_boost_round': 100,
    'verbose': 1
}


dtrain = lgb.Dataset(dp_x_train, dp_y_train)
dvalid = lgb.Dataset(dp_x_test, dp_y_test, reference=dtrain)

bst = lgb.train(
    params, dtrain, valid_sets=dvalid, verbose_eval=10,
    callbacks=[lgb.reset_parameter(learning_rate=lambda current_round: lr*(lr_decay**current_round))],
)

def dp_fe(df):
    df['perc'] = df['count']/df['lenn']
    df['prob_item_domain'] = df['prob_domain']*df['prob_item']
    df['prob_item_text'] = df['prob_text']*df['prob_item']
    
    return df

def generate_datapoint_without_y(candidates, v, domains_10, text_10, proba_domains_10, proba_text_10):

    nunique = len(np.unique(v))
    lenn = len(v)
    counts = counter(v)
    rec_score = get_recurrent_datapoint(v)
    prices = [0] if len(v) == 0 else [item_info[i][0] for i in v]
    maxx = np.max(prices)
    minn = np.min(prices)
    amp = maxx - minn
    
    try:
        unique_domains = get_unique_domains(v)
    except:
        unique_domains = 1

    datapoints = []
    for item in candidates:
        datapoints.append({
            'is_first': 1 if len(v) > 0 and item == v[0] else 0,
            'is_last': 1 if len(v) > 0 and item == v[-1] else 0,
            'count': counts.get(item, 0),
            'nunique': nunique,
            'max_price_viewed': maxx,
            'min_price_viewed': minn,
            'amp_price': amp,
            'lenn': lenn,
            'rec_score': rec_score.get(item, 0),
            'prob_domain': get_domain_score(domain[item], domains_10, proba_domains_10),
            'prob_text': get_domain_score(domain[item], text_10, proba_text_10),
            'prob_item': get_item_score(item, domain[item]),
            'unique_domains': unique_domains,
            'price': item_info[item][0],
            'condition': item_info[item][1]
        })

    datapoints = pd.DataFrame().from_records(datapoints)
    datapoints = dp_fe(datapoints)
    return datapoints
    
def p(lenn_test, preds_domain, preds_text, proba_domain, proba_text, views):

    outputs = []
    for idx in tqdm(range(lenn_test)):
        v = views[idx]
        domains_10 = preds_domain[idx]
        texts_10 = preds_text[idx]
        proba_domains_10 = proba_domain[idx]
        proba_text_10 = proba_text[idx]

        output = []
        j = 0
        while len(output) < 10:
            if len(v) > 0:
#             if proba_domains_10[j] >= 0.0003808080808080808:
                current_domain = domains_10[j]
            else:
                current_domain = texts_10[j]
            current_domain_items = domain_inverse_sorted[current_domain]

            views_of_domain = unique([i for i in v if domain[i] == current_domain])

            current_domain_items = [i for i in current_domain_items if i in views_of_domain]

            output.extend(current_domain_items)

            j += 1
            if j == 10:
                k = 0
                while len(output) < 10:
                    if len(v) > 0:
                        current_domain = domains_10[k]
                    else:
                        current_domain = texts_10[k]
                    current_domain_items = domain_inverse_sorted[current_domain]

                    for i in current_domain_items:
                        if i not in output:
                            output.append(i)
                    k += 1

        output = output[:10]

        datapoints = generate_datapoint_without_y(output, v, domains_10, texts_10, proba_domains_10, proba_text_10)
        preds = bst.predict(datapoints, num_iteration=bst.best_iteration)
        args = preds.argsort()
        output = list(np.array(output)[args[::-1]][:10])

        outputs.append(output)
        
    return outputs
print(len(preds_domain), len(preds_text), len(proba_domain), len(proba_text), len(views), len(uh))
outputs = p(len(test[:1000]), preds_domain[len(train):], preds_text[len(train):], proba_domain[len(train):], proba_text[len(train):], views[len(train):])
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)
average_NDCG(outputs, y_test, domain)<jupyter_output>100%|██████████| 1000/1000 [00:00<00:00, 27811.84it/s]<jupyter_text>Format output<jupyter_code>for i in outputs:
    assert len(i) == 10
    assert len(set(i)) == 10
with open("output_.csv", "w") as f:
    for row in outputs:
        f.write("%s\n" % ','.join(str(col) for col in row))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13713463577161897, 'fraction_numerical': 0.021808268968650613, 'mean_word_length': 3.158375314861461, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '267510', 'license': 'MIT', 'n_tokens_mistral': 5098, 'n_tokens_neox': 4833, 'n_words': 901, 'path': 'ml-295.ipynb', 'repo_name': 'farzinshams/meli-2020-challenge', 'size': '13206'}