<commit_before>import sys

import numpy as np
import numpy.linalg as la

from rlscore.utilities import decomposition
import unittest
from rlscore.learner import LabelRankRLS
from rlscore.kernel import LinearKernel

def mapQids(qids):
    """Maps qids to running numbering starting from zero, and partitions
    the training data indices so that each partition corresponds to one
    query"""
    #Used in FileReader, rls_predict
    qid_dict = {}
    folds = {}
    qid_list = []
    counter = 0
    for index, qid in enumerate(qids):
        if not qid in qid_dict:
            qid_dict[qid] = counter
            folds[qid] = []
            counter += 1
        folds[qid].append(index)
    indslist = []
    for f in folds.values():
        indslist.append(f)
    return indslist

class Test(unittest.TestCase):
    
    def testLabelRankRLS(self):
        
        print
        print
        print
        print
        print "Testing the cross-validation routines of the LabelRankRLS module."
        print
        print
        
        np.random.seed(100)
        floattype = np.float64
        
        m, n = 100, 400 #data, features
        Xtrain = np.mat(np.random.rand(m, n))
        K = Xtrain * Xtrain.T
        ylen = 1
        Y = np.mat(np.zeros((m, ylen), dtype=floattype))
        Y[:, 0] = np.sum(Xtrain, 1)
        
        
        objcount = 20
        labelcount = 5
        
        hoindices = range(labelcount)
        hocompl = list(set(range(m)) - set(hoindices))
        
        size = m
        
        P = np.mat(np.zeros((m, objcount), dtype=np.float64))
        Q = np.mat(np.zeros((objcount, m), dtype=np.float64))
        qidlist = [0 for i in range(100)]
        for h in range(5, 12):
            qidlist[h] = 1
        for h in range(12, 32):
            qidlist[h] = 2
        for h in range(32, 34):
            qidlist[h] = 3
        for h in range(34, 85):
            qidlist[h] = 4
        for h in range(85, 100):
            qidlist[h] = 5
        qidlist_cv = qidlist[5: len(qidlist)]
        
        objcount = max(qidlist) + 1
        P = np.mat(np.zeros((m, objcount), dtype=np.float64))
        for i in range(m):
            qid = qidlist[i]
            P[i, qid] = 1.
        labelcounts = np.sum(P, axis=0)
        D = np.mat(np.zeros((1, m), dtype=np.float64))
        for i in range(m):
            qid = qidlist[i]
            D[0, i] = labelcounts[0, qid]
        L = np.multiply(np.eye(m), D) - P * P.T
        
        Kcv = K[np.ix_(hocompl, hocompl)]
        Ycv = Y[hocompl]
        Ktest = K[np.ix_(hoindices, hocompl)]
        Lcv = L[np.ix_(hocompl, hocompl)]<commit_msg>RankRLS tester now also uses query-wise normalization.<commit_after>import sys

import numpy as np
import numpy.linalg as la

from rlscore.utilities import decomposition
import unittest
from rlscore.learner import LabelRankRLS
from rlscore.kernel import LinearKernel

def mapQids(qids):
    """Maps qids to running numbering starting from zero, and partitions
    the training data indices so that each partition corresponds to one
    query"""
    #Used in FileReader, rls_predict
    qid_dict = {}
    folds = {}
    qid_list = []
    counter = 0
    for index, qid in enumerate(qids):
        if not qid in qid_dict:
            qid_dict[qid] = counter
            folds[qid] = []
            counter += 1
        folds[qid].append(index)
    indslist = []
    for f in folds.values():
        indslist.append(f)
    return indslist

class Test(unittest.TestCase):
    
    def testLabelRankRLS(self):
        
        print
        print
        print
        print
        print "Testing the cross-validation routines of the LabelRankRLS module."
        print
        print
        
        np.random.seed(100)
        floattype = np.float64
        
        m, n = 100, 400 #data, features
        Xtrain = np.mat(np.random.rand(m, n))
        K = Xtrain * Xtrain.T
        ylen = 1
        Y = np.mat(np.zeros((m, ylen), dtype=floattype))
        Y[:, 0] = np.sum(Xtrain, 1)
        
        
        objcount = 20
        labelcount = 5
        
        hoindices = range(labelcount)
        hocompl = list(set(range(m)) - set(hoindices))
        
        size = m
        
        P = np.mat(np.zeros((m, objcount), dtype=np.float64))
        Q = np.mat(np.zeros((objcount, m), dtype=np.float64))
        qidlist = [0 for i in range(100)]
        for h in range(5, 12):
            qidlist[h] = 1
        for h in range(12, 32):
            qidlist[h] = 2
        for h in range(32, 34):
            qidlist[h] = 3
        for h in range(34, 85):
            qidlist[h] = 4
        for h in range(85, 100):
            qidlist[h] = 5
        qidlist_cv = qidlist[5: len(qidlist)]
        
        objcount = max(qidlist) + 1
        P = np.mat(np.zeros((m, objcount), dtype=np.float64))
        for i in range(m):
            qid = qidlist[i]
            P[i, qid] = 1.
        labelcounts = np.sum(P, axis=0)
        P = np.divide(P, np.sqrt(labelcounts))
        D = np.mat(np.ones((1, m), dtype=np.float64))
        L = np.multiply(np.eye(m), D) - P * P.T
        
        Kcv = K[np.ix_(hocompl, hocompl)]
        Ycv = Y[hocompl]
        Ktest = K[np.ix_(hoindices, hocompl)]
        Lcv = L[np.ix_(hocompl, hocompl)]
{'fraction_non_alphanumeric': 0.11241706161137441, 'fraction_numerical': 0.023507109004739336, 'mean_word_length': 1.7522170057381325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7581107', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1798, 'n_words': 472}