<jupyter_start><jupyter_text>Chapter 1: EDA- estimates of location: mean, p-trimmed mean, median- estimates of variability- exploring the distribution- exploring binary and categorical data- correlation and scatter plot- exploring 2 or more variables<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats as stats

sns.set(style="darkgrid")

tips = sns.load_dataset("tips")
tips.head()
# estimates of location: where is the data typically located
# is the estimator robust vs sensitive to outliers?

# mean is sum by count
mean = tips['tip'].sum() / tips['tip'].count()
# trimmed mean is the mean, ignoring the lowest and highest p percentiles
p = .10
n = len(tips['tip'])
from_index = int(n*p)
to_index = int(n*(1-p))
trimmed_mean = tips['tip'].sort_values().iloc[from_index:to_index].mean()
# weighted mean: weigh by inverse of party size 
weighted_mean = sum(tips['tip'] / tips['size']) / sum(1/tips['size'])
# median is 50th percentile value
median = tips['tip'].sort_values().iloc[n//2]

print(f'mean {mean:.2f}')
print(f'trimmed_mean {trimmed_mean:.2f}')
print(f'weighted_mean {weighted_mean:.2f}')
print(f'median {median:.2f}')

# estimates of variability (aka dispersion)
# is the data mostly all together, or widely spread?
# "At the heart of statistics lies variability: 
# measuring it, reducing it, distinguishing random from real variability, 
# identifying the various sources of real variability, 
# and making decisions in the presence of it."

# mean absolute deviation: mean of L1 distance from mean
mean_abs_dev = np.mean(
    np.abs(
        tips['tip'] - tips['tip'].mean()
    )
)

# standard deviation: sqrt of mean of L2 distance from mean
std_dev = np.sqrt(
    np.mean(
        np.square(
            tips['tip'] - tips['tip'].mean()
        )
    )
)

# taking mean is biased - dividing by n-1 is unbiased
# n-1, and not n is the degrees of freedom for SD, 
# since SD computation involves the mean
n = len(tips['tip'])
unbiased_std_dev = np.sqrt(std_dev**2 * n / (n-1))

# median abs dev: median of L1 distance from median
# robust against outliers, unlike SD
median_abs_dev = np.median(
    np.abs(
        tips['tip'] - np.median(tips['tip'])
    )
)

# interquarile range: difference between p25 and p75
low, high = np.percentile(tips['tip'], [25, 75])
interquartile_range = high - low

print(f'mean_abs_dev {mean_abs_dev:.3f}')
print(f'std_dev {std_dev:.3f}')
print(f'unbiased_std_dev {unbiased_std_dev:.3f}')
print(f'median_abs_dev {median_abs_dev:.3f}')
print(f'interquartile_range {interquartile_range:.3f}')

# mathematically: stdev > mean_abs_dev > median_abs_dev
# exploring distribution of the data
# boxplot, frequency table, histogram, density plot

# describe
print('****description****\n', tips['tip'].describe())

# boxplot 
#tips.boxplot('tip', by='size')
# fig, axs = plt.subplots(ncols=3)
sns.boxplot(x='size', y='tip', data=tips)

# histogram and density plot
sns.distplot(tips['tip'])

# bin then histogram of bins
tips['tip_bin'] = pd.cut(tips['tip'], 6)
binned_counts = tips.groupby('tip_bin').agg({'tip':'count'}).reset_index()
sns.barplot(x='tip_bin',y='tip', data=binned_counts)
# exploring binary and categorical data
mode = tips["tip"].mode()[0]
print(f'most frequent tipped amount is: {mode}')
bill_ev = tips['total_bill'].mean()
print(f'expected value of bill is: {bill_ev:.2f}')
# tipping occurrences by day of week
tips.groupby('day').agg({'tip':'count'}).plot(kind='bar')
# correlation and scatterplot
r,pval = stats.pearsonr(tips['tip'], tips['total_bill'])
print(f'pearson corr between tip and bill: {r:.2f}, p={pval:.2f}')
r,pval = stats.spearmanr(tips['tip'], tips['total_bill'])
print(f'spearman corr between tip and bill: {r:.2f}, p={pval:.2f}')

sns.scatterplot(x="total_bill", y="tip", hue="sex", data=tips)
sns.jointplot(
    x='total_bill', 
    y='tip', 
    data=tips, 
    kind='hex', 
    gridsize=12
) # see https://stackoverflow.com/a/29909033 to add legend

sns.jointplot(
    x='total_bill', 
    y='tip', 
    data=tips, 
    kind='kde'
) # https://lmicke.github.io/whisky-post.html
# two categ vars: crosstab
tips.groupby(['size', 'time'])['tip'].count().unstack().fillna(0)
# same as pd.crosstab(tips['size'],tips['time'])
# see also https://pbpython.com/pandas-crosstab.html
# boxplot https://lmicke.github.io/whisky-post.html
sns.boxplot(x='day', y='tip', data=tips)
# violin plot https://lmicke.github.io/whisky-post.html
sns.violinplot(x='day', y='tip', data=tips)
g = sns.FacetGrid(tips, row="sex", col="time", margin_titles=True)
bins = np.linspace(0, tips['tip'].max(), 10)
g.map(plt.hist, 'tip', bins=bins)
# https://stackoverflow.com/a/31385996
def hexbin(x, y, color, **kwargs):
    cmap = sns.light_palette(color, as_cmap=True)
    plt.hexbin(x, y, gridsize=6, cmap=cmap, **kwargs)
    
g = sns.FacetGrid(tips, row='sex', col="time")
min_tip, max_tip = tips['tip'].min(), tips['tip'].max()
min_bill, max_bill = tips['total_bill'].min(), tips['total_bill'].max()
g.map(hexbin, 
      "total_bill", 
      "tip", 
      extent=[min_bill, max_bill, min_tip, max_tip]
     )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16402013942680094, 'fraction_numerical': 0.012393493415956624, 'mean_word_length': 4.94361334867664, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '335771', 'license': 'Unlicense', 'n_tokens_mistral': 1973, 'n_tokens_neox': 1846, 'n_words': 465, 'path': 'practical_ds/practical_stats_ch01_eda.ipynb', 'repo_name': 'gentimouton/dives', 'size': '5164'}