<jupyter_start><jupyter_text>Simple examples  from wikibased on https://en.wikipedia.org/wiki/Discounted_cumulative_gain<jupyter_code>swp = None

data = pd.DataFrame({'userid': [0,0,0,0,0,0,0,0],
                     'movieid': [0,1,2,3,4,5,6,7],
                    'rating':[3, 2, 3, 0, 1, 2, 3, 2]})
recs = np.array([[0,1,2,3,4,5]])
hsz = data.shape[0]
topk = recs.shape[1]
shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)
rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', None, 'rating')
discm, idisc = get_ndcr_discounts(rankm, evalm, topk)
get_ndcg_score(ehits, discm, idisc, alternative=False)<jupyter_output><empty_output><jupyter_text>the result is slightly worse (expected value is 0.785), as normalization is based on the full holdout, not just topk elements  this is an intentional behavior in order to support NDCL score calculation when switch_positive is set  hand-crafted example<jupyter_code>swp = 3

data = pd.DataFrame({'userid': [0,0, 1,1, 2,2],
                     'movieid': [0,1, 2,3, 4,5],
                    'rating':[2,3, 1,3, 5,4]})
recs = np.array([[1,0], [2,3], [5,4]])
hsz = 2
topk = recs.shape[1]
shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)
data.set_index(['userid', 'movieid']).sort_index()
if swp is None:
    is_positive = None
else:
    is_positive = data.rating>=swp
rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', is_positive, 'rating')
discm, idisc = get_ndcr_discounts(rankm, evalm, topk)
get_ndcg_score(ehits, discm, idisc, alternative=False)
get_ndcl_score(emiss, discm, idisc, swp, alternative=False)<jupyter_output><empty_output><jupyter_text>Movielens<jupyter_code>ml_data = get_movielens_data()
ml_data.head()
dm = RecommenderData(ml_data, 'userid', 'movieid', 'rating', seed=0)
dm.get_configuration()
dm.random_holdout = True
dm.prepare()
svd = SVDModel(dm)
svd.rank = 50
svd.build()
swp = 4

svd.switch_positive = swp
data = dm.test.holdout
recs = svd.recommendations
hsz = dm.holdout_size
topk = recs.shape[1]
shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)
if swp is None:
    is_positive = None
else:
    is_positive = (data.rating>=swp).values
rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', is_positive, 'rating')
evalm
ehits
emiss
discm, idisc = get_ndcr_discounts(rankm, evalm, topk)
discm
idisc
get_ndcg_score(ehits, discm, idisc, alternative=False)
get_ndcl_score(emiss, discm, idisc, swp, alternative=False)
get_mrr_score(hrank)<jupyter_output><empty_output><jupyter_text>compare with previous implementation<jupyter_code>def get_matched_predictions(eval_data, holdout_size, recs):
    userid, itemid = 'userid', 'movieid'
    holdout_data = eval_data[itemid]
    holdout_matrix = holdout_data.values.reshape(-1, holdout_size).astype(np.int64)

    matched_predictions = (recs[:, :, None] == holdout_matrix[:, None, :])
    return matched_predictions

def get_feedback_data(eval_data, holdout_size):
    feedback = 'rating'
    eval_data = eval_data[feedback].values
    feedback_data = eval_data.reshape(-1, holdout_size)
    return feedback_data

def get_rnkng_scores(eval_data, holdout_size, recs, switch_positive=None, alternative=False):
    matched_predictions = get_matched_predictions(eval_data, holdout_size, recs)
    feedback_data = get_feedback_data(eval_data, holdout_size)
    
    users_num, topk, holdout = matched_predictions.shape
    ideal_scores_idx = np.argsort(feedback_data, axis=1)[:, ::-1] #returns column index only
    ideal_scores_idx = np.ravel_multi_index((np.arange(feedback_data.shape[0])[:, None],
                                             ideal_scores_idx), dims=feedback_data.shape)
        
    where = np.where
    is_positive = feedback_data >= switch_positive
    positive_feedback = where(is_positive, feedback_data, 0)
    negative_feedback = where(~is_positive, feedback_data-switch_positive, 0)
    
    relevance_scores_pos = (matched_predictions * positive_feedback[:, None, :]).sum(axis=2)
    relevance_scores_neg = (matched_predictions * negative_feedback[:, None, :]).sum(axis=2)
    ideal_scores_pos = positive_feedback.ravel()[ideal_scores_idx]
    ideal_scores_neg = negative_feedback.ravel()[ideal_scores_idx]
    
    if alternative:
        relevance_scores_pos = 2**relevance_scores_pos - 1
        relevance_scores_neg = 2.0**relevance_scores_neg - 1
        ideal_scores_pos = 2**ideal_scores_pos - 1
        ideal_scores_neg = 2.0**ideal_scores_neg - 1

    disc_num = max(topk, holdout)
    discount = np.log2(np.arange(2, disc_num+2))            
    dcg = (relevance_scores_pos / discount[:topk]).sum(axis=1)
    dcl = (relevance_scores_neg / -discount[:topk]).sum(axis=1)
    idcg = (ideal_scores_pos / discount[:holdout]).sum(axis=1)
    idcl = (ideal_scores_neg / -discount[:holdout]).sum(axis=1)
    
    with np.errstate(invalid='ignore'):
        ndcg = np.nansum(dcg / idcg) / users_num
        ndcl = np.nansum(dcl / idcl) / users_num

    ranking_score = namedtuple('Ranking', ['nDCG', 'nDCL'])._make([ndcg, ndcl])
    return ranking_score
get_rnkng_scores(data, hsz, recs, switch_positive=swp, alternative=False)
get_ranking_scores(rankm, hrank, mrank, evalm, ehits, emiss, switch_positive=swp, topk=topk, alternative=False)
svd.evaluate('hits', not_rated_penalty=None)
svd.evaluate('relevance')
from polara.recommender import defaults
defaults.ndcg_alternative = False
svd.evaluate('ranking')
svd.evaluate('ranking', topk=1)<jupyter_output><empty_output><jupyter_text>Hand-picked test<jupyter_code>test_user = 98
test_data = svd.data.test.holdout.query('userid=={}'.format(test_user))
test_recs = svd.recommendations[test_user, :]
topk = len(test_recs)
print(test_recs)
test_data
test_data.loc[:, 'movieid'].isin(test_recs)
(rankm, hrank, mrank,
 evalm, ehits, emiss) = assemble_scoring_matrices(test_recs, test_data,
                                                  svd._key, svd._target,
                                                  (test_data.rating>=swp).values, feedback='rating')
hrank.data
hrank.indices
ehits.data
ehits.indices
discm, idisc = get_ndcr_discounts(rankm, evalm, topn=2)
discm.data
discm.indices
idisc.data
idisc.indices<jupyter_output><empty_output><jupyter_text>NDCG<jupyter_code>get_ndcg_score(ehits, discm, idisc, alternative=False)
print('rec rank', np.where(np.isin(test_recs, test_data.movieid))[0] + 1)
print('rec item', test_recs[np.isin(test_recs, test_data.movieid)])<jupyter_output>rec rank [1 9]
rec item [1045 1130]<jupyter_text>NDCL<jupyter_code>emiss.data
emiss.indices
emiss.multiply(discm).data
emiss.multiply(idisc)
get_ndcl_score(emiss, discm, idisc, swp, alternative=False)<jupyter_output><empty_output><jupyter_text>Why normalization in NDCG is changed basically due to NDCL metric, which is "the lower the better"  this means that ideal score is 0 regular case<jupyter_code>cg = lambda rel, pos: rel / np.log2(1+pos)

print('dcg ', cg(5, 9))
print('idcg', cg(5, 1) + cg(5, 2))
print('ndcg', cg(5, 9) / (cg(5, 1) + cg(5, 2)))<jupyter_output>dcg  1.505149978319906
idcg 8.154648767857287
ndcg 0.1845756967795682<jupyter_text>singular, but still ok<jupyter_code>cl = lambda rel, pos: (np.exp(rel-4)-1) / (-np.log2(1+pos))

print('dcl ', 0)
print('idcl', 0)
with np.errstate(invalid='ignore'):
    print('ndcl', np.array([0.]) / np.array([0.]))<jupyter_output>dcl  0
idcl 0
ndcl [nan]<jupyter_text>broken case  when dcl is above zere and idcl is exactly 0 (due to only topk selected result, where negatove examples are not included at all)<jupyter_code>cl = lambda rel, pos: (np.exp(rel-4)-1) / (-np.log2(1+pos))

print('dcl ', cl(3, 3))
print('idcl', 0)
with np.errstate(invalid='ignore'): # will not catch an error
    print('ndcl', cl(3, 3) / np.array([0.]))<jupyter_output>dcl  0.31606027941427883
idcl 0
ndcl [inf]
{'fraction_non_alphanumeric': 0.1396789280748325, 'fraction_numerical': 0.028062191884717484, 'mean_word_length': 4.953348382242288, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '621555', 'license': 'Apache-2.0', 'n_tokens_mistral': 3209, 'n_tokens_neox': 2999, 'n_words': 578, 'path': 'recsys19_hybridsvd/polara-master/tests/polara_new_evaluation.ipynb', 'repo_name': 'chenzheng128/SoRec', 'size': '7911'}