<jupyter_start><jupyter_text>Estimation<jupyter_code>def hmetric(y_true, y_pred):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    return 2 * tp * tn / (2*tp*tn + tp*fp + tn * fn)
param_grid = {
    "use_positive": [True, False],
    "alpha": [0.02, 0.05, 0.1, 0.5],
    "beta": [-0.2, -0.1, 0, 0.1, 0.2]
}
grid = GridSearchCV(LazyPatternClassifier(), param_grid,
                    scoring={'f1': 'f1',
                             'accuracy': 'accuracy',
                             'precision': 'precision',
                             'recall': 'recall',
                             'hmetric': make_scorer(hmetric)},
                    cv=KFold(n_splits=3, shuffle=True, random_state=495),
                    verbose=4, return_train_score=True,
                    n_jobs=1, refit=False)
loaders = [
    #('Breast Cancer', get_breast_cancer),
    ('Heart Disease', get_heart_disease),
    #('Mammographic Mass', get_mammographic_mass),
    #('Seismic Bumps', get_seismic_bumps),
    #('Titanic', get_titanic),
    #('Breast GSE', get_breast_GSE),
    #('Liver GSE', get_liver_GSE),
    #('Prostate GSE', get_prostate_GSE),
]
result = {}
for name, loader in loaders:
    X, y = loader()
    grid.fit(X, y)
    result[name] = grid.cv_results_.copy()
    print('Completed:', name)
with open('results.pkl', 'wb') as f:
    pickle.dump(result, f)
!chmod -w 'results.pkl'<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>with open('results.pkl', 'rb') as f:
    result = pickle.load(f)
frames = {key: pd.DataFrame(value) for key, value in result.items()}
for key, value in frames.items():
    value['dataset'] = key
metaframe = pd.concat(frames.values(), sort=True).reset_index(drop=True)
!rm -f metaframe.csv
metaframe.to_csv('metaframe.csv', index=False)
!chmod -w metaframe.csv
metaframe = pd.read_csv('metaframe.csv')
metaframe.sample(5)
params = [
    'param_use_positive',
    #'param_alpha',
    #'param_beta',
]

metrics = [
    f'mean_{data}_{metric}'
    for data in ['test', 'train']
    for metric in ['f1', 'accuracy', 'precision', 'recall', 'hmetric']
]

foi = params + metrics + ['mean_fit_time', 'mean_score_time', 'dataset']
df = metaframe[foi].copy()
for param in params:
    df = df.set_index(param, drop=True, append=(param is not params[0]))
target = 'mean_test_hmetric'
df2 = df[[target, 'dataset']]

df3 = df2.iloc[:df2.shape[0] // df2.dataset.nunique()].copy()
df3.drop(columns=df3.columns, inplace=True)
for dataset in df2.dataset.unique():
    df3[dataset] = df2[target].values[(df2.dataset == dataset).values]

df3 = df3.sort_index().round(4)
datasets = ['Breast Cancer', 'Heart Disease', 'Mammographic Mass', 'Seismic Bumps',
        'Titanic', 'Breast GSE', 'Liver GSE', 'Prostate GSE']
for d in datasets:
    if d not in df3:
        df3[d] = np.nan
df3 = df3[datasets]
df3
print(df3.to_latex())<jupyter_output>\begin{tabular}{lrrrrrrrr}
\toprule
{} &  Breast Cancer &  Heart Disease &  Mammographic Mass &  Seismic Bumps &  Titanic &  Breast GSE &  Liver GSE &  Prostate GSE \\
param\_use\_positive &                &                &                    &                &          &             &            &               \\
\midrule
False              &            0.0 &            0.0 &             0.6167 &          0.000 &   0.1759 &         NaN &        NaN &           NaN \\
True               &            0.0 &            0.0 &             0.6536 &          0.018 &   0.4012 &         NaN &        NaN &           NaN \\
\bottomrule
\end{tabular}<jupyter_text>Plots<jupyter_code>data = {
    "Метод ближайшего\nсоседа": {'path':'./results/model_kNN', 'n_params': 2},
    "Логистическая\nрегрессия": {'path':'./results/model_LR', 'n_params': 2},
    "Случайный лес": {'path':'./results/model_RF', 'n_params': 2},
    "Алгоритм ленивой\nклассификации": {'path':'./results/A1', 'n_params': 1},
    "Алгоритм 1": {'path':'./results/A2', 'n_params': 3},
    "Алгоритм 2": {'path':'./results/A3', 'n_params': 3},
    "Алгоритм 3": {'path':'./results/A4', 'n_params': 3},
}
labels = list(data.keys())
datasets = ['Breast Cancer', 'Heart Disease', 'Mammographic Mass', 'Seismic Bumps',
        'Titanic',]
frames = {
    name: pd.read_csv(
        os.path.join(data_['path'], 'h_value.csv'),
        index_col=list(range(data_['n_params'])),
    )
    for name, data_ in data.items()
}
def get_h_values(series):
    if isinstance(series.index, pd.MultiIndex) and series.index.names[0] == 'param_use_positive':
        mask = series.index.codes[0].values().astype(bool)
        return series[mask].max(), series[~mask].max()
    else:
        return (series.max(), 0)
x = np.arange(len(labels))
width = 0.35

for dataset in datasets:
    h_values = [get_h_values(frame[dataset]) for frame in frames.values()]
    h_values, h_values_dual = zip(*h_values)
    
    plt.figure(figsize=(12,8))
    rects1 = plt.bar(x - width/2, h_values, width, label='$H$-метрика')
    rects2 = plt.bar(x + width/2, h_values_dual, width, label='$H$-метрика двойств. версий')

    for i, (h1, h2) in enumerate(zip(h_values, h_values_dual)):
        plt.text(i-width/2, h1, str(h1.round(2)), va='bottom', ha='center')
        if h2 != 0:
            plt.text(i+width/2, h2, str(h2.round(2)), va='bottom', ha='center')
    
    plt.ylabel('$H$')
    plt.ylim(0, 1)
    plt.title(dataset)
    plt.xticks(x, labels)
    plt.legend()
    plt.grid()
    plt.hlines(max(max(h_values), max(h_values_dual)), *plt.xlim(), 'k', linestyles='--')

    plt.gcf().tight_layout()
    plt.savefig(dataset + '.pdf')
    plt.show()
    plt.clf()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17457203994293866, 'fraction_numerical': 0.023181169757489302, 'mean_word_length': 2.7720242098184262, 'pattern_counts': {'":': 10, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '366809', 'license': 'MIT', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2099, 'n_words': 406, 'path': 'main.ipynb', 'repo_name': 'D-Sokol/lazy-pattern-classifier', 'size': '5608'}