<jupyter_start><jupyter_text>An approach to use confidence for improving the performance of models<jupyter_code># An example about using the confidence to improve the performance of models
from sklearn import datasets
from tests import *
from sklearn.model_selection import train_test_split as ts
import numpy as np
import warnings
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
warnings.filterwarnings('ignore')

np.random.seed(456)

# save res
score_with_confidence_series = []
score_without_confidence_series = []

# import data
breast_cancer = datasets.load_breast_cancer()
    
# Get features as the X matrix form
X = breast_cancer.data
Y = breast_cancer.target
# Split data into trian and test parts
X_train, X_test, Y_train, Y_test = ts(X, Y, test_size=0.5)

for intensity in range(10, 50):
    intensity *= .01
    print('intensity %.2f' % intensity, end=' : ')
    # Added noise to the test data (generate the gap of data)
    X_test_with_noise = X_test + intensity * np.random.random(X_test.shape)

    # Get the confidences
    train_confidence, test_confidence = detector.LR_test(X_train, X_test_with_noise, max_iter=1e3, confidence_score=True)
    train_confidence, test_confidence = train_confidence.reshape(-1, 1), test_confidence.reshape(-1, 1)
    # train_confidence *= .1
    # test_confidence *= .1
    
    # Add confidences into the raw data
    X_train_with_confidence = np.concatenate([X_train, train_confidence], axis=1)
    X_test_with_confidence = np.concatenate([X_test_with_noise, test_confidence], axis=1)

    # Create model
    clf = DecisionTreeClassifier(max_depth=None)
    # clf = LogisticRegression(max_iter=1e3)

    # Using data without confidence to train
    clf.fit(X_train, Y_train)
    score_without_confidence = clf.score(X_test_with_noise, Y_test)
    score_without_confidence_series.append(score_without_confidence)

    # Using data without confidence to train
    clf.fit(X_train_with_confidence, Y_train)
    score_with_confidence = clf.score(X_test_with_confidence, Y_test)
    score_with_confidence_series.append(score_with_confidence)
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure

figure(figsize=(12, 8), dpi=80)

print('Score without confidence: %.2f' % np.mean(score_without_confidence_series))
print('Score with confidence: %.2f' % np.mean(score_with_confidence_series))
plt.plot(score_without_confidence_series, label='Without confidence')
plt.plot(score_with_confidence_series, label='With confidence')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08532818532818533, 'fraction_numerical': 0.011969111969111969, 'mean_word_length': 5.542929292929293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '609864', 'license': 'Apache-2.0', 'n_tokens_mistral': 871, 'n_tokens_neox': 809, 'n_words': 203, 'path': 'Tutorial.ipynb', 'repo_name': 'JonnesLin/distribution_detection', 'size': '2590'}