<jupyter_start><jupyter_text>Tukey Box Plot & IQR Method<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>df = pd.read_csv("iris_dataset.txt", header=None, names=['sepal_length', 'sepal_width', 'petal_length'
                                                         ,'petal_width', 'type'])<jupyter_output><empty_output><jupyter_text>Data info<jupyter_code>#pd.options.display.float_format='{:.3f}'.format
df.describe()<jupyter_output><empty_output><jupyter_text>Visualize using Box Plot<jupyter_code>plt.figure(figsize=(12, 8))
df.boxplot(return_type='dict')

plt.plot();<jupyter_output><empty_output><jupyter_text>On column "sepal_width" there are ~4 values out of boundaries  Tukey IQR Method<jupyter_code>q1 = df["sepal_width"].quantile(0.25)
q3 = df["sepal_width"].quantile(0.75)
iqr = q3-q1

upper_threshold = q3+(iqr*1.5)
lower_threshold = q1-(iqr*1.5)
print( 'Lower Threshold:'+str(lower_threshold)+'\nUpper Threshold:'+str(upper_threshold))
outliers= df.loc[(df['sepal_width'] > upper_threshold) | (df['sepal_width'] < lower_threshold)]
outliers
def filter_iqr(x):
    if (x < 2.05 or x > 4.05):
        return "Outlier"
    else:
        return "Normal"
df['outliers_based_on_IQR'] = df.apply(lambda row: filter_iqr(row['sepal_width']), axis=1)
df.head(20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14939242315939957, 'fraction_numerical': 0.021443888491779844, 'mean_word_length': 5.896551724137931, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '561733', 'license': 'MIT', 'n_tokens_mistral': 554, 'n_tokens_neox': 530, 'n_words': 80, 'path': 'Tukey_IQR.ipynb', 'repo_name': 'LeoTserr/Tukey_IQR', 'size': '1399'}