<jupyter_start><jupyter_text>outlier detectionuse IsolationForest to predict outliers<jupyter_code># importing libaries ----
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pylab import savefig
from sklearn.ensemble import IsolationForest

# default plot settings
plt.rcParams['figure.figsize'] = [15, 10]

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning) 
warnings.filterwarnings("ignore", category=FutureWarning) 
# Generating data ----

rng = np.random.RandomState(42)

# Generating training data 
X_train = 0.2 * rng.randn(200, 2)
X_train = np.r_[X_train + 3, X_train]
X_train = pd.DataFrame(X_train, columns = ['x1', 'x2'])

# Generating new, 'normal' observation
X_test = 0.2 * rng.randn(200, 2)
X_test = np.r_[X_test + 3, X_test]
X_test = pd.DataFrame(X_test, columns = ['x1', 'x2'])

# Generating outliers
X_outliers = rng.uniform(low=-1, high=5, size=(25, 2))
X_outliers = pd.DataFrame(X_outliers, columns = ['x1', 'x2'])
# use scipy and treat all outliers as more than 2.5 std dev's from Z score
from scipy import stats
df_zscore = X_outliers[(np.abs(stats.zscore(X_outliers)) < 2.5).all(axis=1)]

# Plotting generated data ----

plt.title("Data")

p1 = plt.scatter(X_train.x1, X_train.x2, c='white',
                 s=20*4, edgecolor='k')
p2 = plt.scatter(X_test.x1, X_test.x2, c='blue',
                 s=20*4, edgecolor='k')
p3 = plt.scatter(X_outliers.x1, X_outliers.x2, c='yellow',
                s=20*4, edgecolor='k')

p4 = plt.scatter(df_zscore.x1, df_zscore.x2, c='red',
                s=10, edgecolor='k')

plt.axis('tight')
plt.xlim((-2, 5))
plt.ylim((-2, 5))
plt.legend([p1, p2, p3, p4],
           ["training observations",
            "new regular obs.", 
            "new abnormal obs.",
            "stddev"],
           loc="lower right")

plt.show()
# Isolation Forest ----

# training the model
clf = IsolationForest(random_state=rng)
clf.fit(X_train)

# predictions
y_pred_train = clf.predict(X_train)
y_pred_test = clf.predict(X_test)

# Returns -1 for outliers and 1 for inliers.

y_pred_outliers = clf.predict(X_outliers)
# new, 'normal' observations
print("Accuracy:", list(y_pred_test).count(1)/y_pred_test.shape[0])
# outliers
print("Accuracy:", list(y_pred_outliers).count(-1)/y_pred_outliers.shape[0])
# adding the predicted label
X_outliers = X_outliers.assign(y = y_pred_outliers)

plt.title("Outlier Inspection")

p1 = plt.scatter(X_train.x1, X_train.x2, c='white',
                 s=20*4, edgecolor='k')
p2 = plt.scatter(X_outliers.loc[X_outliers.y == -1, ['x1']], 
                 X_outliers.loc[X_outliers.y == -1, ['x2']], 
                 c='blue', s=20*4, edgecolor='k')
p3 = plt.scatter(X_outliers.loc[X_outliers.y == 1, ['x1']], 
                 X_outliers.loc[X_outliers.y == 1, ['x2']], 
                 c='yellow', s=20*4, edgecolor='k')

plt.axis('tight')
plt.xlim((-2, 5))
plt.ylim((-2, 5))
plt.legend([p1, p2, p3],
           ["training observations",
            "detected outliers", 
            "false positive outlier"],
           loc="lower right")

plt.show();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16159025328630972, 'fraction_numerical': 0.03270278935556268, 'mean_word_length': 3.5814977973568283, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '262490', 'license': 'MIT', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1187, 'n_words': 244, 'path': 'outliers.ipynb', 'repo_name': 'darenr/outlier_detection', 'size': '3119'}