<commit_before>import numpy as np
import scipy.sparse as sp
import warnings
#import pdb

# Matrix-vector product wrapper
# A is a numpy 2d array or matrix, or a scipy matrix or sparse matrix.
# x is a numpy vector only.
# Compute A.dot(x) if t is False,
#         A.transpose().dot(x)  otherwise.
def mult(A,x,t=False):
  if(sp.issparse(A)):
    m = A.shape[0]
    n = A.shape[1]
    if(t):
      return(np.array(sp.csr_matrix(x).dot(A).transpose().todense()).reshape(n))
    return(np.array(A.dot(sp.csr_matrix(x).transpose()).todense()).reshape(m))
  if(t):
    return(x.dot(A))
  return(A.dot(x))

def orthog(Y,X):
  """Orthogonalize a vector or matrix Y against the columns of the matrix X.
  This function requires that the column dimension of Y is less than X and
  that Y and X have the same number of rows.
  """
  dotY = mult(X,Y,t=True)
  return (Y - mult(X,dotY))

# Simple utility function used to check linear dependencies during computation:
def invcheck(x):
  eps2  = 2*np.finfo(np.float).eps
  if(x>eps2):
    x = 1/x
  else:
    x = 0
    warnings.warn("Ill-conditioning encountered, result accuracy may be poor")
  return(x)

def irlb(A,n,tol=0.0001,maxit=50):
  """Estimate a few of the largest singular values and corresponding singular<commit_msg>Cleaned up sparse matrix mul function<commit_after>import numpy as np
import scipy.sparse as sp
import warnings
#import pdb

# Matrix-vector product wrapper
# A is a numpy 2d array or matrix, or a scipy matrix or sparse matrix.
# x is a numpy vector only.
# Compute A.dot(x) if t is False,
#         A.transpose().dot(x)  otherwise.
def mult(A,x,t=False):
  if(sp.issparse(A)):
    m = A.shape[0]
    n = A.shape[1]
    if(t):
      return(sp.csr_matrix(x).dot(A).transpose().todense().A[:,0])
    return(A.dot(sp.csr_matrix(x).transpose()).todense().A[:,0])
  if(t):
    return(x.dot(A))
  return(A.dot(x))

def orthog(Y,X):
  """Orthogonalize a vector or matrix Y against the columns of the matrix X.
  This function requires that the column dimension of Y is less than X and
  that Y and X have the same number of rows.
  """
  dotY = mult(X,Y,t=True)
  return (Y - mult(X,dotY))

# Simple utility function used to check linear dependencies during computation:
def invcheck(x):
  eps2  = 2*np.finfo(np.float).eps
  if(x>eps2):
    x = 1/x
  else:
    x = 0
    warnings.warn("Ill-conditioning encountered, result accuracy may be poor")
  return(x)

def irlb(A,n,tol=0.0001,maxit=50):
  """Estimate a few of the largest singular values and corresponding singular
{'fraction_non_alphanumeric': 0.13033175355450238, 'fraction_numerical': 0.01263823064770932, 'mean_word_length': 3.65625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48227', 'n_tokens_mistral': 930, 'n_tokens_neox': 906, 'n_words': 288}