<commit_before>import gc 
import numpy
import logging
import itertools
import scipy.sparse.linalg
import exp.util.SparseUtils as ExpSU
import numpy.testing as nptst 
from sppy import csarray 
from exp.sandbox.RandomisedSVD import RandomisedSVD
from exp.util.MCEvaluator import MCEvaluator
from apgl.util.Util import Util
from apgl.util.Parameter import Parameter
from exp.sandbox.recommendation.AbstractMatrixCompleter import AbstractMatrixCompleter
from exp.util.SparseUtilsCython import SparseUtilsCython
from exp.sandbox.SVDUpdate import SVDUpdate
from exp.util.LinOperatorUtils import LinOperatorUtils
from exp.util.SparseUtils import SparseUtils

def learnPredict(args): 
    """
    A function to train on a training set and test on a test set, for a number 
    of values of rho. 
    """
    learner, trainX, testX, rhos = args 
    logging.debug("k=" + str(learner.getK()))
    logging.debug(learner) 
    
    testInds = testX.nonzero()
    trainXIter = []
    testIndList = []    
    
    for rho in rhos: 
        trainXIter.append(trainX)
        testIndList.append(testInds)
    
    trainXIter = iter(trainXIter)

    ZIter = learner.learnModel(trainXIter, iter(rhos))
    predXIter = learner.predict(ZIter, testIndList)
    
    errors = numpy.zeros(rhos.shape[0])
    for j, predX in enumerate(predXIter): 
        errors[j] = MCEvaluator.rootMeanSqError(testX, predX)
        logging.debug("Error = " + str(errors[j]))
        del predX 
        gc.collect()
        
    return errors 

class IterativeSoftImpute(AbstractMatrixCompleter):
    """
    Given a set of matrices X_1, ..., X_T find the completed matrices.
    """
    def __init__(self, rho=0.1, eps=0.01, k=None, svdAlg="propack", updateAlg="initial", logStep=10, kmax=None, postProcess=False, p=50, q=2, weighted=False, verbose=False, qu=1):
        """
        Initialise imputing algorithm with given parameters. The rho is a value
        for use with the soft thresholded SVD. Eps is the convergence threshold and
        k is the rank of the SVD.

        :param rho: The regularisation parameter for soft-impute in [0, 1] (lambda = rho * maxSv)

        :param eps: The convergence threshold

        :param k: The number of SVs to compute

        :param svdAlg: The algorithm to use for computing a low rank + sparse matrix

        :param updateAlg: The algorithm to use for updating an SVD for a new matrix
        
        :param p: The oversampling used for the randomised SVD
        
        :param q: The exponent used for the randomised SVD 
        """
        super(AbstractMatrixCompleter, self).__init__()

        self.rho = rho
        self.eps = eps
        self.k = k
        self.svdAlg = svdAlg
        self.updateAlg = updateAlg
        self.p = p
        self.q = q
        #The q used for the SVD update 
        self.qu = qu
        if k != None:
            self.kmax = k*5
        else:
            self.kmax = None
        self.logStep = logStep
        self.postProcess = postProcess 
        self.postProcessSamples = 10**6
        self.maxIterations = 30
        self.weighted = weighted 
        self.verbose = verbose 

    def learnModel(self, XIterator, rhos=None):
        """
        Learn the matrix completion using an iterator which outputs
        a sequence of sparse matrices X. The output of this method is also
        an iterator which outputs a sequence of completed matrices in factorised 
        form. 
        
        :param XIterator: An iterator which emits scipy.sparse.csc_matrix objects 
        
        :param rhos: An optional array of rhos for model selection using warm restarts 
        """

        class ZIterator(object):
            def __init__(self, XIterator, iterativeSoftImpute):
                self.tol = 10**-6
                self.j = 0
                self.XIterator = XIterator
                self.iterativeSoftImpute = iterativeSoftImpute
                self.rhos = rhos 

            def __iter__(self):
                return self
            
            def next(self):
                X = self.XIterator.next()
                logging.debug("Learning on matrix with shape: " + str(X.shape) + " and " + str(X.nnz) + " non-zeros")    
                
                if self.iterativeSoftImpute.weighted: 
                    #Compute row and col probabilities 
                    up, vp = SparseUtils.nonzeroRowColsProbs(X)
                    nzuInds = up==0
                    nzvInds = vp==0
                    u = numpy.sqrt(1/(up + numpy.array(nzuInds, numpy.int))) 
                    v = numpy.sqrt(1/(vp + numpy.array(nzvInds, numpy.int)))
                    u[nzuInds] = 0 
                    v[nzvInds] = 0 
                
                if self.rhos != None: 
                    self.iterativeSoftImpute.setRho(self.rhos.next())

                if not scipy.sparse.isspmatrix_csc(X):
                    raise ValueError("X must be a csc_matrix not " + str(type(X)))
                    
                #Figure out what lambda should be 
                #PROPACK has problems with convergence 
                Y = scipy.sparse.csc_matrix(X, dtype=numpy.float)
                U, s, V = ExpSU.SparseUtils.svdArpack(Y, 1, kmax=20)
                del Y
                #U, s, V = SparseUtils.svdPropack(X, 1, kmax=20)
                maxS = s[0]
                logging.debug("Largest singular value : " + str(maxS))

                (n, m) = X.shape

                if self.j == 0:
                    self.oldU = numpy.zeros((n, 1))
                    self.oldS = numpy.zeros(1)
                    self.oldV = numpy.zeros((m, 1))
                else:
                    oldN = self.oldU.shape[0]
                    oldM = self.oldV.shape[0]

                    if self.iterativeSoftImpute.updateAlg == "initial":
                        if n > oldN:
                            self.oldU = Util.extendArray(self.oldU, (n, self.oldU.shape[1]))
                        elif n < oldN:
                            self.oldU = self.oldU[0:n, :]

                        if m > oldM:
                            self.oldV = Util.extendArray(self.oldV, (m, self.oldV.shape[1]))
                        elif m < oldN:
                            self.oldV = self.oldV[0:m, :]
                    elif self.iterativeSoftImpute.updateAlg == "svdUpdate":
                        pass
                    elif self.iterativeSoftImpute.updateAlg == "zero":
                        self.oldU = numpy.zeros((n, 1))
                        self.oldS = numpy.zeros(1)
                        self.oldV = numpy.zeros((m, 1))
                    else:
                        raise ValueError("Unknown SVD update algorithm: " + self.updateAlg)

                rowInds, colInds = X.nonzero()

                gamma = self.iterativeSoftImpute.eps + 1
                i = 0

                self.iterativeSoftImpute.measures = numpy.zeros((self.iterativeSoftImpute.maxIterations, 4))

                while gamma > self.iterativeSoftImpute.eps:
                    if i == self.iterativeSoftImpute.maxIterations: 
                        logging.debug("Maximum number of iterations reached")
                        break 
                    
                    ZOmega = SparseUtilsCython.partialReconstructPQ((rowInds, colInds), self.oldU*self.oldS, self.oldV)
                    Y = X - ZOmega
                    #Y = Y.tocsc()
                    #del ZOmega
                    Y = csarray.fromScipySparse(Y, storagetype="row")
                    gc.collect()

                    if self.iterativeSoftImpute.svdAlg=="propack":
                        L = LinOperatorUtils.sparseLowRankOp(Y, self.oldU, self.oldS, self.oldV, parallel=False)                        
                        newU, newS, newV = SparseUtils.svdPropack(L, k=self.iterativeSoftImpute.k, kmax=self.iterativeSoftImpute.kmax)
                        newS = newS + numpy.random.randn(newS.shape[0])*10**-8 
                    elif self.iterativeSoftImpute.svdAlg=="arpack":
                        L = LinOperatorUtils.sparseLowRankOp(Y, self.oldU, self.oldS, self.oldV, parallel=False)                        <commit_msg>noise change <commit_after>import gc 
import numpy
import logging
import itertools
import scipy.sparse.linalg
import exp.util.SparseUtils as ExpSU
import numpy.testing as nptst 
from sppy import csarray 
from exp.sandbox.RandomisedSVD import RandomisedSVD
from exp.util.MCEvaluator import MCEvaluator
from apgl.util.Util import Util
from apgl.util.Parameter import Parameter
from exp.sandbox.recommendation.AbstractMatrixCompleter import AbstractMatrixCompleter
from exp.util.SparseUtilsCython import SparseUtilsCython
from exp.sandbox.SVDUpdate import SVDUpdate
from exp.util.LinOperatorUtils import LinOperatorUtils
from exp.util.SparseUtils import SparseUtils

def learnPredict(args): 
    """
    A function to train on a training set and test on a test set, for a number 
    of values of rho. 
    """
    learner, trainX, testX, rhos = args 
    logging.debug("k=" + str(learner.getK()))
    logging.debug(learner) 
    
    testInds = testX.nonzero()
    trainXIter = []
    testIndList = []    
    
    for rho in rhos: 
        trainXIter.append(trainX)
        testIndList.append(testInds)
    
    trainXIter = iter(trainXIter)

    ZIter = learner.learnModel(trainXIter, iter(rhos))
    predXIter = learner.predict(ZIter, testIndList)
    
    errors = numpy.zeros(rhos.shape[0])
    for j, predX in enumerate(predXIter): 
        errors[j] = MCEvaluator.rootMeanSqError(testX, predX)
        logging.debug("Error = " + str(errors[j]))
        del predX 
        gc.collect()
        
    return errors 

class IterativeSoftImpute(AbstractMatrixCompleter):
    """
    Given a set of matrices X_1, ..., X_T find the completed matrices.
    """
    def __init__(self, rho=0.1, eps=0.01, k=None, svdAlg="propack", updateAlg="initial", logStep=10, kmax=None, postProcess=False, p=50, q=2, weighted=False, verbose=False, qu=1):
        """
        Initialise imputing algorithm with given parameters. The rho is a value
        for use with the soft thresholded SVD. Eps is the convergence threshold and
        k is the rank of the SVD.

        :param rho: The regularisation parameter for soft-impute in [0, 1] (lambda = rho * maxSv)

        :param eps: The convergence threshold

        :param k: The number of SVs to compute

        :param svdAlg: The algorithm to use for computing a low rank + sparse matrix

        :param updateAlg: The algorithm to use for updating an SVD for a new matrix
        
        :param p: The oversampling used for the randomised SVD
        
        :param q: The exponent used for the randomised SVD 
        """
        super(AbstractMatrixCompleter, self).__init__()

        self.rho = rho
        self.eps = eps
        self.k = k
        self.svdAlg = svdAlg
        self.updateAlg = updateAlg
        self.p = p
        self.q = q
        #The q used for the SVD update 
        self.qu = qu
        if k != None:
            self.kmax = k*5
        else:
            self.kmax = None
        self.logStep = logStep
        self.postProcess = postProcess 
        self.postProcessSamples = 10**6
        self.maxIterations = 30
        self.weighted = weighted 
        self.verbose = verbose 

    def learnModel(self, XIterator, rhos=None):
        """
        Learn the matrix completion using an iterator which outputs
        a sequence of sparse matrices X. The output of this method is also
        an iterator which outputs a sequence of completed matrices in factorised 
        form. 
        
        :param XIterator: An iterator which emits scipy.sparse.csc_matrix objects 
        
        :param rhos: An optional array of rhos for model selection using warm restarts 
        """

        class ZIterator(object):
            def __init__(self, XIterator, iterativeSoftImpute):
                self.tol = 10**-6
                self.j = 0
                self.XIterator = XIterator
                self.iterativeSoftImpute = iterativeSoftImpute
                self.rhos = rhos 

            def __iter__(self):
                return self
            
            def next(self):
                X = self.XIterator.next()
                logging.debug("Learning on matrix with shape: " + str(X.shape) + " and " + str(X.nnz) + " non-zeros")    
                
                if self.iterativeSoftImpute.weighted: 
                    #Compute row and col probabilities 
                    up, vp = SparseUtils.nonzeroRowColsProbs(X)
                    nzuInds = up==0
                    nzvInds = vp==0
                    u = numpy.sqrt(1/(up + numpy.array(nzuInds, numpy.int))) 
                    v = numpy.sqrt(1/(vp + numpy.array(nzvInds, numpy.int)))
                    u[nzuInds] = 0 
                    v[nzvInds] = 0 
                
                if self.rhos != None: 
                    self.iterativeSoftImpute.setRho(self.rhos.next())

                if not scipy.sparse.isspmatrix_csc(X):
                    raise ValueError("X must be a csc_matrix not " + str(type(X)))
                    
                #Figure out what lambda should be 
                #PROPACK has problems with convergence 
                Y = scipy.sparse.csc_matrix(X, dtype=numpy.float)
                U, s, V = ExpSU.SparseUtils.svdArpack(Y, 1, kmax=20)
                del Y
                #U, s, V = SparseUtils.svdPropack(X, 1, kmax=20)
                maxS = s[0]
                logging.debug("Largest singular value : " + str(maxS))

                (n, m) = X.shape

                if self.j == 0:
                    self.oldU = numpy.zeros((n, 1))
                    self.oldS = numpy.zeros(1)
                    self.oldV = numpy.zeros((m, 1))
                else:
                    oldN = self.oldU.shape[0]
                    oldM = self.oldV.shape[0]

                    if self.iterativeSoftImpute.updateAlg == "initial":
                        if n > oldN:
                            self.oldU = Util.extendArray(self.oldU, (n, self.oldU.shape[1]))
                        elif n < oldN:
                            self.oldU = self.oldU[0:n, :]

                        if m > oldM:
                            self.oldV = Util.extendArray(self.oldV, (m, self.oldV.shape[1]))
                        elif m < oldN:
                            self.oldV = self.oldV[0:m, :]
                    elif self.iterativeSoftImpute.updateAlg == "svdUpdate":
                        pass
                    elif self.iterativeSoftImpute.updateAlg == "zero":
                        self.oldU = numpy.zeros((n, 1))
                        self.oldS = numpy.zeros(1)
                        self.oldV = numpy.zeros((m, 1))
                    else:
                        raise ValueError("Unknown SVD update algorithm: " + self.updateAlg)

                rowInds, colInds = X.nonzero()

                gamma = self.iterativeSoftImpute.eps + 1
                i = 0

                self.iterativeSoftImpute.measures = numpy.zeros((self.iterativeSoftImpute.maxIterations, 4))

                while gamma > self.iterativeSoftImpute.eps:
                    if i == self.iterativeSoftImpute.maxIterations: 
                        logging.debug("Maximum number of iterations reached")
                        break 
                    
                    ZOmega = SparseUtilsCython.partialReconstructPQ((rowInds, colInds), self.oldU*self.oldS, self.oldV)
                    Y = X - ZOmega
                    #Y = Y.tocsc()
                    #del ZOmega
                    Y = csarray.fromScipySparse(Y, storagetype="row")
                    gc.collect()

                    if self.iterativeSoftImpute.svdAlg=="propack":
                        L = LinOperatorUtils.sparseLowRankOp(Y, self.oldU, self.oldS, self.oldV, parallel=False)                        
                        newU, newS, newV = SparseUtils.svdPropack(L, k=self.iterativeSoftImpute.k, kmax=self.iterativeSoftImpute.kmax)
                        newS = newS + numpy.random.randn(newS.shape[0])*10**-7 
                    elif self.iterativeSoftImpute.svdAlg=="arpack":
                        L = LinOperatorUtils.sparseLowRankOp(Y, self.oldU, self.oldS, self.oldV, parallel=False)                        
{'fraction_non_alphanumeric': 0.08694590304066431, 'fraction_numerical': 0.0070826718769080475, 'mean_word_length': 1.6068756963234123, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '522953', 'n_tokens_mistral': 5011, 'n_tokens_neox': 4637, 'n_words': 1198}