<jupyter_start><jupyter_text>Standard deviation and mean<jupyter_code>print("Mean ",np.mean(distribution))
print("STD ",np.std(distribution))
def select_outliers_std(distribution):
    mean = np.mean(distribution)
    std = np.std(distribution)
    list_outliers = []
    for i in distribution:
        if i > (mean+(3*std)) or i < (mean-(3*std)):
            list_outliers.append(i)
    return list_outliers
select_outliers_std(distribution)<jupyter_output><empty_output><jupyter_text>IQR Rule<jupyter_code>def select_outliers(distribution):
    median_data = np.median(distribution)
    Q1 = np.percentile(distribution, 25, interpolation="midpoint")
    Q3 = np.percentile(distribution, 75, interpolation="midpoint")
    IQR = Q3 - Q1
    list_outliers = []
    for v in distribution:
        if v > (median_data + (1.5 * IQR)) or v < (median_data - (1.5 * IQR)):
            list_outliers.append(v)
    return list_outliers
list_res = select_outliers(distribution)
list_res.sort()
list_res<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1232876712328767, 'fraction_numerical': 0.0136986301369863, 'mean_word_length': 4.441489361702128, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '398249', 'license': 'MIT', 'n_tokens_mistral': 382, 'n_tokens_neox': 341, 'n_words': 65, 'path': 'notebooks/7. Outlier detection/IQR Rule.ipynb', 'repo_name': 'danilopcarlotti/nlp4law', 'size': '1022'}