<jupyter_start><jupyter_text>Reliability Diagram<jupyter_code># fop = The proportion of samples whose class is the positive class, in each bin (fraction of positives).
# mpv = The mean predicted probability in each bin.
fop, mpv = calibration_curve(testy, probs, n_bins = 10, normalize= True)
len(fop)
len(mpv)
pyplot.plot([0,1], [0, 1], linestyle= '--')
pyplot.plot(mpv, fop, marker = '.')
pyplot.show()<jupyter_output><empty_output><jupyter_text>Calibration<jupyter_code>from sklearn.calibration import calibration_curve, CalibratedClassifierCV
model = SVC(random_state= 42)
calibrated = CalibratedClassifierCV(model, method= 'sigmoid', cv = 5)
calibrated.fit(trainX, trainy)
probs = calibrated.predict_proba(testX)[:, 1]
fop, mpv = calibration_curve(testy, probs, n_bins = 10, normalize = True)
pyplot.plot([0,1], [0, 1], linestyle= '--')
pyplot.plot(mpv, fop, marker = '.')
pyplot.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14332247557003258, 'fraction_numerical': 0.01737242128121607, 'mean_word_length': 7.087719298245614, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '485505', 'license': 'MIT', 'n_tokens_mistral': 366, 'n_tokens_neox': 340, 'n_words': 79, 'path': 'probability_calibration/probability_calibration_example.ipynb', 'repo_name': 'nightfuryml/ml-topics', 'size': '921'}