<commit_before>
import numpy
import logging
import sys
import scipy.sparse.linalg
from apgl.util.Sampling import Sampling 
from apgl.util.ProfileUtils import ProfileUtils
from exp.util.SparseUtils import SparseUtils
from exp.util.LinOperatorUtils import LinOperatorUtils

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

class LinOperatorUtilsProfile(object):
    def __init__(self):
        numpy.random.seed(21)        
        
        #Create a low rank matrix  
        n = 100000 
        m = 100000 
        self.r = 200 
        k = 10**6
        
        self.X = SparseUtils.generateSparseLowRank((n, m), self.r, k)

    def profileParallelSparseOp(self):
        L = LinOperatorUtils.parallelSparseOp(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = numpy.random.rand(self.X.shape[0])
                q = numpy.random.rand(self.X.shape[1])
                
                L.matvec(q)
                L.rmatvec(p)
        
        ProfileUtils.profile('run()', globals(), locals())

    def profileParallelSparseOp2(self):
        L = LinOperatorUtils.parallelSparseOp(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = 200
                W = numpy.random.rand(self.X.shape[1], p)
                
                L.matmat(W)
        
        ProfileUtils.profile('run()', globals(), locals())
        
        
    def profileAsLinearOperator(self):
        L = scipy.sparse.linalg.aslinearoperator(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = numpy.random.rand(self.X.shape[0])
                q = numpy.random.rand(self.X.shape[1])
                
                L.matvec(q)
                L.rmatvec(p)
        
        ProfileUtils.profile('run()', globals(), locals())

    def profileAsLinearOperator2(self):
        L = scipy.sparse.linalg.aslinearoperator(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = 200
                W = numpy.random.rand(self.X.shape[1], p)
                
                L.matmat(W)
        
        ProfileUtils.profile('run()', globals(), locals())
        
profiler = LinOperatorUtilsProfile()
profiler.profileParallelSparseOp()
#profiler.profileAsLinearOperator()
#profiler.profileParallelSparseOp2() # 78.77
#profiler.profileAsLinearOperator2() #44 <commit_msg>Test matmat on large matrices <commit_after>
import numpy
import logging
import sys
import scipy.sparse.linalg
from apgl.util.Sampling import Sampling 
from apgl.util.ProfileUtils import ProfileUtils
from exp.util.SparseUtils import SparseUtils
from exp.util.LinOperatorUtils import LinOperatorUtils

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

class LinOperatorUtilsProfile(object):
    def __init__(self):
        numpy.random.seed(21)        
        
        #Create a low rank matrix  
        n = 100000 
        m = 100000 
        self.r = 200 
        k = 10**8
        
        self.X = SparseUtils.generateSparseLowRank((n, m), self.r, k)

    def profileParallelSparseOp(self):
        L = LinOperatorUtils.parallelSparseOp(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = numpy.random.rand(self.X.shape[0])
                q = numpy.random.rand(self.X.shape[1])
                
                L.matvec(q)
                L.rmatvec(p)
        
        ProfileUtils.profile('run()', globals(), locals())

    def profileParallelSparseOp2(self):
        L = LinOperatorUtils.parallelSparseOp(self.X)
        
        p = 300
        W = numpy.random.rand(self.X.shape[1], p)        
        
        def run(): 
            numRuns = 1
            for i in range(numRuns): 
                L.matmat(W)
        
        ProfileUtils.profile('run()', globals(), locals())
        
        
    def profileAsLinearOperator(self):
        L = scipy.sparse.linalg.aslinearoperator(self.X)
        
        def run(): 
            numRuns = 10 
            for i in range(numRuns): 
                p = numpy.random.rand(self.X.shape[0])
                q = numpy.random.rand(self.X.shape[1])
                
                L.matvec(q)
                L.rmatvec(p)
        
        ProfileUtils.profile('run()', globals(), locals())

    def profileAsLinearOperator2(self):
        L = scipy.sparse.linalg.aslinearoperator(self.X)
        
        p = 300
        W = numpy.random.rand(self.X.shape[1], p)
        
        def run(): 
            numRuns = 1 
            for i in range(numRuns): 
                L.matmat(W)
        
        ProfileUtils.profile('run()', globals(), locals())
        
profiler = LinOperatorUtilsProfile()
#An advantage to be parallel when nnz > 10^8 
profiler.profileAsLinearOperator2() #42s 
#profiler.profileParallelSparseOp2() #37s

{'fraction_non_alphanumeric': 0.10427110487266894, 'fraction_numerical': 0.01985161419691197, 'mean_word_length': 1.6875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3943475', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1485, 'n_words': 295}