<jupyter_start><jupyter_text>Detecting outliers<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
tips_data = sns.load_dataset('tips')
tips_data.head()
# Box plot shows the outliers clearly, hence, plotting it
sns.boxplot(tips_data['tip']).set_title('Box plot of the tips received')
plt.show()
sns.boxplot(tips_data['total_bill']).set_title('Box plot of the total bill received')
plt.show()
# Function to detect outliers by the Inter Quartile Range method
def detect_outliers(data, lb, ub):
    outliers = []
    for i in data:
        if i<=lb or i>=ub:
            outliers.append(i)
    return outliers
# Getting the 25% and 75% values for the inter quartile range of the 'tip' column
Q1, Q3 = np.percentile(tips_data['tip'], [25, 75])
print('Q1 = ', Q1)
print('Q3 = ', Q3)

# Calculating the Inter Quartile Range (middle 50% of the data)
IQR = Q3 - Q1
print('IQR = ', IQR)

# Getting the lower and upper bounds to get outliers beyond these values
lb = Q1 - (1.5*IQR)
ub = Q3 + (1.5*IQR)
print('Lower bound = ', lb)
print('Upper bound = ', ub)

# Applying the function to 'tip' column
outliers = detect_outliers(tips_data['tip'], lb, ub)
y = set(outliers)
print('Outliers of the tip column are: ',y)
print('Number of outliers of the tip column are: ',len(y))
# Getting the 25% and 75% values for the inter quartile range of the 'total_bill' column
Q1, Q3 = np.percentile(tips_data['total_bill'], [25, 75])
print('Q1 = ', Q1)
print('Q3 = ', Q3)

# Calculating the Inter Quartile Range (middle 50% of the data)
IQR = Q3 - Q1
print('IQR = ', IQR)

# Getting the lower and upper bounds to get outliers beyond these values
lb = Q1 - (1.5*IQR)
ub = Q3 + (1.5*IQR)
print('Lower bound = ', lb)
print('Upper bound = ', ub)

# Applying the function to 'total_bill' column
outliers = detect_outliers(tips_data['total_bill'], lb, ub)
y = set(outliers)
print('Outliers of the total_bill column are: ', y)
print('Number of outliers of the total_bill column are: ', len(y))
# Function to detect outliers by the z - factor method
def calculate_outliers(data, threshold):
    outliers = []
    std = np.std(data)
    mean = np.mean(data)
    print(std, mean)
    for num in data:
        z = (num-mean) / std # calculating the z factor
        
        if z > threshold: # 
            outliers.append(num)
            
    return set(outliers)
# Applying the function to 'tip' column
tip_outliers = calculate_outliers(tips_data['tip'], 2.1) # 3 is generally taken as threshold, can alter it as needed
print('Outliers of tip column = ', tip_outliers)
print('Number of outliers of tip column = ', len(tip_outliers))
# Applying the function to 'total_bill' column
total_bill_outliers = calculate_outliers(tips_data['total_bill'], 2.3)
print('Outliers of tip column = ', tip_outliers)
print('Number of outliers of total_bill column = ', len(total_bill_outliers))
# Making a scatter plot to detect outliers from this graph
sns.relplot(x='total_bill', y='tip', data=tips_data)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12049770792403405, 'fraction_numerical': 0.017354289456450556, 'mean_word_length': 4.303819444444445, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '654436', 'license': 'MIT', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1006, 'n_words': 355, 'path': 'Session_6.ipynb', 'repo_name': 'reeha-parkar/applied-statistics', 'size': '3054'}