<jupyter_start><jupyter_text>Logistic<jupyter_code>from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import TimeSeriesSplit

f, ax = plt.subplots(1, 3, figsize=(15,4))
i = 0
for stock in stocks:
    log_clf = lob.logistic_regression(d_stocks[stock], 0, len(d_stocks[stock]))
    lob.plot_learning_curve(
        log_clf, d_stocks[stock][['prev_queue_imbalance', 'queue_imbalance']],
        d_stocks[stock]['mid_price_indicator'], 
        cv=TimeSeriesSplit(n_splits=2), ax=ax[i], scoring='f1')
    ax[i].set_title('Learning Curve for {}'.format(stock))
    i += 1
if should_save_fig:
    plt.savefig('logistic_regression_learning_curves_prev_imb.png')
def logistic_regression(df):
    clf = LogisticRegression()
    X = df[['prev_queue_imbalance', 'queue_imbalance']]
    y = df['mid_price_indicator']
    clf.fit(X, y)
    return clf
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(21,6))
for stock in stocks:
    log_clf = logistic_regression(d_stocks[stock])
    pred_train = log_clf.predict(d_stocks[stock][['prev_queue_imbalance', 'queue_imbalance']])
    pred_test = log_clf.predict(d_test_stocks[stock][['prev_queue_imbalance', 'queue_imbalance']])
    d_stocks[stock]['pred_log'] = pred_train
    d_test_stocks[stock]['pred_log'] = pred_test
    print(len(d_stocks[stock]), len(d_test_stocks[stock]) )
    lob.plot_f1(d_stocks[stock], log_clf, stock=stock, label='', ax=ax1,
                 title='Precision vs Recall for 3 stocks on training data', 
                features=['prev_queue_imbalance', 'queue_imbalance'])
    lob.plot_f1(d_test_stocks[stock], log_clf, stock=stock, label='', ax=ax2,
                 title='Precision vs Recall for 3 stocks on testing data',
                features=['prev_queue_imbalance', 'queue_imbalance'])
if should_save_fig:
    plt.savefig('logistic_regression_precision_vs_recall_prev_imb.png')
def get_scores_dict_for_data(functions_to_run, dfs, clf, stock):
    scores = {'stock': stock}
    for func_name, func in functions_to_run.items():
        for df_name, df in dfs.items():
            pred = clf.predict(df[['prev_queue_imbalance', 'queue_imbalance']])
            scores['{}_{}'.format(df_name, func_name)] = func(df['mid_price_indicator'], pred)
    return scores
            
functions_to_run = {'precision': metrics.precision_score, 'roc_auc': metrics.roc_auc_score,
                   'f1_score': metrics.f1_score, 'recall': metrics.recall_score,  
                   'matthews': metrics.matthews_corrcoef, 'kappa': metrics.cohen_kappa_score}
scores = []
for stock in stocks:
    log_clf = logistic_regression(d_stocks[stock])
    dfs = {'test': d_test_stocks[stock]}
    res = get_scores_dict_for_data(functions_to_run, dfs, log_clf, stock)
    
    res_train = model.validate_model(LogisticRegression(), d_stocks[stock][['queue_imbalance']], 
                         d_stocks[stock]['mid_price_indicator'])
    res = {**res, **res_train}
    scores.append(res)
df_scores = pd.DataFrame(scores, index=stocks)
for i in range(len(stocks)):
    plt.plot(df_scores['matthews'].values[i], label=stocks[i])
plt.legend()
def convert_scores(df, column):
    scores = []
    for i, row in df.iterrows():
        scores.append(np.mean(row[column]))
    return scores
scores_columns = ['f1', 'kappa', 'matthews', 'precision', 'recall', 'roc_auc', 'train_f1', 'train_kappa',
       'train_matthews', 'train_precision', 'train_recall', 'train_roc_auc']

for col in scores_columns:
    df_scores[col] = convert_scores(df_scores, col)
df_scores
print(df_scores[['train_matthews', 'matthews', 'test_matthews', 
                 'train_roc_auc', 'roc_auc', 'test_roc_auc']].to_latex())
print(df_scores[['f1', 'test_f1_score', 'precision', 'test_precision', 'recall', 'test_recall']].to_latex())
df_scores.to_csv('res_overview_all_three_logistic_regression-prev_imb.csv')
f, ax = plt.subplots(1, 3, figsize=(27,6))
for i in range(len(stocks)):
    stock = stocks[i]
    d_stocks[stock]['Predicition of Mid Price Indicator'] = d_stocks[stock]['pred_log']
    d_stocks[stock]['Mid Price Indicator'] = d_stocks[stock]['mid_price_indicator']
    d_stocks[stock][['Predicition of Mid Price Indicator', 'Mid Price Indicator']].plot(
        kind='kde', ax=ax[i])
    ax[i].set_title('Density of Mid Price Indicator and its prediction {} on training data'.format(stock))
    ax[i].legend(loc='lower right')
plt.savefig('density_of_mid_price_and_prediction_training_data_prev_imb.png')
f, ax = plt.subplots(1, 3, figsize=(27,6))
for i in range(len(stocks)):
    stock = stocks[i]
    d_stocks[stock]['Predicition of Mid Price Indicator'] = d_stocks[stock]['pred_log']
    d_stocks[stock]['Mid Price Indicator'] = d_stocks[stock]['mid_price_indicator']
    sns.heatmap(metrics.confusion_matrix(d_stocks[stock]['mid_price_indicator'], d_stocks[stock]['pred_log']), 
               ax=ax[i], annot=True, fmt='d', xticklabels=['Negative', 'Positive'], 
                yticklabels=['Negative', 'Positive'])
    
    ax[i].set_title('{}'.format(stock))
    ax[i].set_ylabel('True Mid Price Indicator')
    ax[i].set_xlabel('Predicted Mid Price Indicator')
if should_save_fig:
    print('Saving figure')
    plt.savefig('logistic_prev_imb_confusion_matrix.png')
f, ax = plt.subplots(3, 1, figsize=(16, 16))
i = 0

for i in range(len(stocks)):
    stock = stocks[i]
    df = d_stocks[stock]
    X = df[['prev_queue_imbalance', 'queue_imbalance']].values
    y = df['mid_price_indicator'].values.astype(np.integer)

    clf = logistic_regression(df)
    # plot_decision_regions(X[0:1000], y[0:1000], clf=clf, ax=ax[i], colors=','.join(['orange', 'blue']))
    ax[i].set_xlabel('Prev Queue Imbalance')
    ax[i].set_ylabel('Queue Imbalance')
    
    ax[i].set_title('Logistic Regression Decision Regions for {} on training data'.format(stock))
    ax[i].set_xlim(-1.01, 1.01)

if should_save_fig:
    plt.savefig('logistic_regression_decision_region_prev_imb.png'.format(s))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14968152866242038, 'fraction_numerical': 0.01039222259470332, 'mean_word_length': 4.8385518590998045, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '48652', 'license': 'MIT', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2156, 'n_words': 367, 'path': 'overview_val10/overview_all_three_logistic_regression-prev_imb.ipynb', 'repo_name': 'vevurka/mt-lob', 'size': '5966'}