<jupyter_start><jupyter_text>Predictions<jupyter_code>def get_kernel_from_method(method):
    return method.split('_')[1]

class CFR():
    def __init__(self, clf, feature_name, is_svm=True):
        self.clf = clf
        self.feature_name = feature_name
        self.is_svm = is_svm

    def get_result(self, gdf_df, should_validate=False, method=None):
        if self.is_svm:
            return gdf_df.train_svm(C=self.clf.C, gamma=self.clf.gamma, coef0=self.clf.coef0, 
                                    kernel=self.clf.kernel, 
                                    feature_name=self.feature_name, should_validate=should_validate)
        else:
            res = gdf_df.train_clf(self.clf, self.feature_name, should_validate=should_validate)
            if method:
                res['kernel'] = method
            return res<jupyter_output><empty_output><jupyter_text>GDF Plain<jupyter_code>df_gdf_plain_res = pd.DataFrame()
gdf_results_dir = '../gaussian_filter/data_res_9061_len15000'
results_files = os.listdir(gdf_results_dir)
results_files = [f for f in results_files if stock in f and 'partial' not in f]
for f in results_files:
    temp_df = pd.read_csv(os.path.join(gdf_results_dir, f))
    df_gdf_plain_res = df_gdf_plain_res.append(temp_df)
print(df_gdf_plain_res.columns)
df_gdf_plain_res.sort_values(by='matthews', ascending=False).groupby('kernel').head(1)
r = 1.0
s = 0.1
C = 1.0 
gamma = 10000.0
filename = 'data_gdf/gdf_{}_len{}_r{}_s{}_K{}.csv'.format(stock, length, r, s, K)<jupyter_output><empty_output><jupyter_text>GDF with PCA<jupyter_code>df_gdf_res = pd.DataFrame()
gdf_results_dir = '../gdf_pca/res_pca_gdf3_15000'
results_files = os.listdir(gdf_results_dir)
results_files = [f for f in results_files if stock in f and 'partial' not in f]
for f in results_files:
    temp_df = pd.read_csv(os.path.join(gdf_results_dir, f))
    df_gdf_res = df_gdf_res.append(temp_df)

df_gdf_res.sort_values(by='matthews', ascending=False).groupby('kernel').head(2)
df_gdf_res = pd.DataFrame()
gdf_results_dir = '../gdf_pca/res_pca1_gdf_que_15000'
results_files = os.listdir(gdf_results_dir)
results_files = [f for f in results_files if stock in f and 'partial' not in f]
for f in results_files:
    temp_df = pd.read_csv(os.path.join(gdf_results_dir, f))
    df_gdf_res = df_gdf_res.append(temp_df)

df_gdf_res.sort_values(by='matthews', ascending=False).groupby('kernel').head(2)
df_gdf_res = pd.DataFrame()
gdf_results_dir = '../gdf_pca/res_pca2_gdf_que_15000'
results_files = os.listdir(gdf_results_dir)
results_files = [f for f in results_files if stock in f and 'partial' not in f]
for f in results_files:
    temp_df = pd.read_csv(os.path.join(gdf_results_dir, f))
    df_gdf_res = df_gdf_res.append(temp_df)

df_gdf_res.sort_values(by='matthews', ascending=False).groupby('kernel').head(2)
df_res1 = pd.DataFrame(res1)
df_res1.sort_values(by='test_matthews', ascending=False).head()
filename_pattern = 'res_{}_mlp.csv'
df_gdf_pca_mlp_res = pd.DataFrame()  
for stock in stocks:
    df_gdf_pca_mlp_res = df_gdf_pca_mlp_res.append(pd.read_csv(filename_pattern.format(stock)))
df_gdf_pca_mlp_res = df_gdf_pca_mlp_res.sort_values(by='matthews', ascending=False)
columns = ['alpha', 'features', 'hidden_layer_size',
       'kernel', 'matthews', 'r', 'roc_auc', 's',
       'stock', 'test_matthews', 'test_roc_auc', 'train_matthews', 
       'train_roc_auc']
df_gdf_pca_mlp_res[df_gdf_pca_mlp_res['features'] == 'pca_gdf_que_prev10'][columns].groupby(
    ['stock', 'r', 's']).head(1).sort_values(by=['matthews', 'test_matthews'], ascending=False)
df = df_gdf_pca_mlp_res[df_gdf_pca_mlp_res['features'] == 'pca_gdf_que_prev10'][columns].groupby(
    ['stock', 'r', 's']).head(1).sort_values(by=['matthews', 'test_matthews'], ascending=False)

fig, ax = plt.subplots(1, 3, figsize=(15, 5), sharex=True, sharey=True)
i = 0
for stock in stocks:
    d = df[df['stock'] == int(stock)]
    ticks = []
    for j, row in d.iterrows():
        ticks.append('r={} s={} '.format(row['r'], row['s']))
    d[['test_matthews', 'matthews']].plot(kind='bar', ax=ax[i])
    
    print(ticks)
    plt.xticks(list(range(len(ticks))), ticks)
    i += 1
plt.tight_layout()
df = df_gdf_pca_mlp_res[df_gdf_pca_mlp_res['features'] == 'pca_gdf_que_prev10'][columns].groupby(
    ['stock', 'r', 's']).head(1).sort_values(by=['matthews', 'test_matthews'], ascending=False)

fig, ax = plt.subplots(3, 2, figsize=(15, 15))
j = 0
i = 0
for stock in stocks:
    d = df[df['stock'] == int(stock)]
    sns.heatmap(data=d[[ 'r', 's', 'matthews']].pivot(index='r', columns='s', values=['matthews'])['matthews'], 
                ax=ax[i][j],
                annot=True)
    ax[i][j].set_title('Train MCC score for {}'.format(stock))
    j +=1
    data = d[[ 'r', 's', 'test_matthews']].pivot(index='r', columns='s', values=['test_matthews'])['test_matthews']
    sns.heatmap(
        data=data, 
        ax=ax[i][j],
        annot=True)
    ax[i][j].set_title('Test MCC score for {}'.format(stock))
    j = 0
    i += 1
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15640467234211047, 'fraction_numerical': 0.01504652544050683, 'mean_word_length': 4.570011025358324, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '195123', 'license': 'MIT', 'n_tokens_mistral': 2134, 'n_tokens_neox': 2042, 'n_words': 329, 'path': 'overview_val10/overview_all_gdf.ipynb', 'repo_name': 'vevurka/mt-lob', 'size': '5051'}