<jupyter_start><jupyter_text>Helper Functions<jupyter_code>from timeit import default_timer

class Timer(object):
    def __init__(self):
        self._timer = default_timer
    
    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args):
        self.stop()

    def start(self):
        """Start the timer."""
        self.start = self._timer()

    def stop(self):
        """Stop the timer. Calculate the interval in seconds."""
        self.end = self._timer()
        self.interval = self.end - self.start
import gzip
def load_data(nrows, ncols, cached = 'data/mortgage.npy.gz'):
    if os.path.exists(cached):
        print('use mortgage data')
        with gzip.open(cached) as f:
            X = np.load(f)
        X = X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]
    else:
        print('use random data')
        X = np.random.rand(nrows,ncols)
    df = pd.DataFrame({'fea%d'%i:X[:,i] for i in range(X.shape[1])})
    return df
from sklearn.metrics import mean_squared_error
def array_equal(a,b,threshold=5e-3,with_sign=True):
    a = to_nparray(a)
    b = to_nparray(b)
    if with_sign == False:
        a,b = np.abs(a),np.abs(b)
    error = mean_squared_error(a,b)
    res = error<threshold
    return res

def to_nparray(x):
    if isinstance(x,np.ndarray) or isinstance(x,pd.DataFrame):
        return np.array(x)
    elif isinstance(x,np.float64):
        return np.array([x])
    elif isinstance(x,cudf.DataFrame) or isinstance(x,cudf.Series):
        return x.to_pandas().values
    return x<jupyter_output><empty_output><jupyter_text>Run tests<jupyter_code>%%time
nrows = 2**22
ncols = 40

X = load_data(nrows,ncols)
print('data',X.shape)
n_components = 10
random_state = 42
%%time
algorithm='arpack'
tsvd_sk = skTSVD(n_components=n_components,algorithm=algorithm, 
            random_state=random_state)
result_sk = tsvd_sk.fit_transform(X)
%%time
X = cudf.DataFrame.from_pandas(X)
%%time
algorithm='full'
tsvd_cuml = cumlTSVD(n_components=n_components,algorithm=algorithm, 
            random_state=random_state)
result_cuml = tsvd_cuml.fit_transform(X)
for attr in ['singular_values_','components_']:
    passed = array_equal(getattr(tsvd_sk,attr),getattr(tsvd_cuml,attr),threshold=0.1)
    # larger error margin due to different algorithms: arpack vs full
    message = 'compare tsvd: cuml vs sklearn {:>25} {}'.format(attr,'equal' if passed else 'NOT equal')
    print(message)
passed = array_equal(result_sk,result_cuml,threshold=0.1)
# larger error margin due to different algorithms: arpack vs full
message = 'compare tsvd: cuml vs sklearn transformed results %s'%('equal'if passed else 'NOT equal')
print(message)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1278084714548803, 'fraction_numerical': 0.00847145488029466, 'mean_word_length': 3.773286467486819, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '174587', 'license': 'Apache-2.0', 'n_tokens_mistral': 980, 'n_tokens_neox': 947, 'n_words': 202, 'path': 'python/notebooks/tsvd_demo.ipynb', 'repo_name': 'rongou/cuml', 'size': '2715'}