<jupyter_start><jupyter_text>importing the MLBox<jupyter_code>from mlbox.preprocessing import *
from mlbox.optimisation import *
from mlbox.prediction import *
sep = ','
paths = ["train.csv","test.csv"]
target_name="Survived"

scoring = 'accuracy'
n_folds=3<jupyter_output><empty_output><jupyter_text>reading and cleaning all files<jupyter_code>rd = Reader(sep)
df = rd.train_test_split(paths, target_name)
dft = Drift_thresholder()
df = dft.fit_transform(df)<jupyter_output>computing drifts...
CPU time: 0.318742990494 seconds

Top 10 drifts

('PassengerId', 1.0)
('Name', 0.99704442935827498)
('Ticket', 0.84579063327276627)
('Cabin', 0.59550700418006364)
('Embarked', 0.53834737054254522)
('SibSp', 0.52134352316754273)
('Fare', 0.51744001298932041)
('Sex', 0.51694417436933204)
('Pclass', 0.51382143975575834)
('Parch', 0.50992009325888499)

deleted variables : ['Name', 'PassengerId', 'Ticket']

dumping drift coefficients into directory : save
drift coefficients dumped<jupyter_text>tuning<jupyter_code>opt = Optimiser(scoring, n_folds)
opt.evaluate(None, df)
space = {
    
        'ne__numerical_strategy':{"search":"choice",
                                 "space":[0]},
        'ce__strategy':{"search":"choice",
                        "space":["label_encoding","random_projection", "entity_embedding"]}, 
        'fs__threshold':{"search":"uniform",
                        "space":[0.01,0.3]},    
        'est__max_depth':{"search":"choice",
                                  "space":[3,4,5,6,7]}
    
        }

best = opt.optimise(space, df,15)
prd = Predictor()
prd.fit_predict(best,df)<jupyter_output>fitting the pipeline...
CPU time: 0.176958799362 seconds

predicting...
CPU time: 0.0571658611298 seconds

top 10 predictions :

        0.0       1.0  Survived_predicted
0  0.951520  0.048480                   0
1  0.915900  0.084100                   0
2  0.929849  0.070151                   0
3  0.744718  0.255282                   0
4  0.736510  0.263490                   0
5  0.680616  0.319384                   0
6  0.636542  0.363458                   0
7  0.953092  0.046908                   0
8  0.168989  0.831011                   1
9  0.981171  0.018829                   0

dumping predictions into directory : save
{'fraction_non_alphanumeric': 0.12472259209942299, 'fraction_numerical': 0.17132711939636042, 'mean_word_length': 2.6006389776357826, 'pattern_counts': {'":': 8, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '355233', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1036, 'n_tokens_neox': 814, 'n_words': 154, 'path': 'examples/classification/example.ipynb', 'repo_name': 'lizh0019/loan', 'size': '2253'}