<jupyter_start><jupyter_text>Test on Large StarAI DSSanity check to assess performance and to make sure I do not make mortal mistakes.<jupyter_code>import mercs
import numpy as np

import os
import pandas as pd

from mercs.core import Mercs
from os.path import dirname<jupyter_output><empty_output><jupyter_text>Setup<jupyter_code>root = dirname(dirname(os.getcwd()))
data = os.path.join(root, 'data', 'step-01')

ds = 'ad'
fns = ['{}-train.csv'.format(ds), '{}-test.csv'.format(ds)]

train = pd.read_csv(os.path.join(data, fns[0]), header=None).values
test = pd.read_csv(os.path.join(data,fns[1]), header=None).values

clf = Mercs(
    max_depth=16,
    selection_algorithm="random",
    fraction_missing=0.3,
    nb_targets=4,
    nb_iterations=4,
    n_jobs=8,
    verbose=1,
    inference_algorithm="dask",
    max_steps=8,
    prediction_algorithm="it",
    random_state=800
)
nominal_attributes = set(range(train.shape[1]))
clf.fit(train, nominal_attributes=nominal_attributes)<jupyter_output>/cw/dtailocal/repos/mercs/src/mercs/algo/induction.py:100: UserWarning: 
        Training is being parallellized using Joblib. Number of jobs = 8
        
  warnings.warn(msg)
[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.
[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    4.1s
[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    7.7s
[Parallel(n_jobs=8)]: Done 434 tasks      | elapsed:   13.3s
[Parallel(n_jobs=8)]: Done 784 tasks      | elapsed:   21.6s
[Parallel(n_jobs=8)]: Done 1234 tasks      | elapsed:   32.1s
[Parallel(n_jobs=8)]: Done 1556 out of 1556 | elapsed:   39.3s finished<jupyter_text>import dill as pklwith open('file-all.pkl', 'wb') as f:    pkl.dump(clf, f)    del clf.m_listwith open('file-nomodels.pkl', 'wb') as f:    pkl.dump(clf, f)    del clf.m_codeswith open('file-nocodes.pkl', 'wb') as f:    pkl.dump(clf, f)    del clf.m_fimpsdel clf.m_scorewith open('file-nonothing.pkl', 'wb') as f:    pkl.dump(clf, f)<jupyter_code>from joblib import dump, load
import blosc
import dill as pkl
with open('dill.pkl', 'wb') as f:
    pkl.dump(clf, f, protocol=4) 
dump(clf, 'model.lz4', compress='lz4') 
clf.m_codes = blosc.pack_array(clf.m_codes)
clf.m_score = blosc.pack_array(clf.m_score)
clf.m_fimps = blosc.pack_array(clf.m_fimps)

%timeit clf = load('model.lz4') 
with open('dill.pkl', 'wb') as f:
    pkl.dump(clf, f, protocol=4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14386109962794544, 'fraction_numerical': 0.030177759404712692, 'mean_word_length': 4.525114155251141, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '547908', 'license': 'MIT', 'n_tokens_mistral': 1025, 'n_tokens_neox': 964, 'n_words': 203, 'path': 'note/integrations/Untitled.ipynb', 'repo_name': 'MattiasDC/mercs', 'size': '2419'}