<jupyter_start><jupyter_text>Normal Distribution- Different displays of normally distributed data- Compare different samples from a normal distribution- Check for normality- Work with the cumulative distribution function (CDF)<jupyter_code>%pylab inline
import scipy.stats as stats

# seaborn is a package for the visualization of statistical data
import seaborn as sns
sns.set(style='ticks')<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>Different Representations<jupyter_code>''' Different aspects of a normal distribution'''
# Generate the data
x = r_[-10:10:0.1]
rv = stats.norm(0,1)   # random variate

x2 = r_[0:1:0.001]

ax = subplot2grid((3,2),(0,0), colspan=2)
plot(x,rv.pdf(x))
xlim([-10,10])
title('Normal Distribution - PDF')

subplot(323)
plot(x,rv.cdf(x))
xlim([-4,4])
title('CDF: cumulative distribution fct')

subplot(324)
plot(x,rv.sf(x))
xlim([-4,4])
title('SF: survival fct')

subplot(325)
plot(x2,rv.ppf(x2))
title('PPF')

subplot(326)
plot(x2,rv.isf(x2))
title('ISF')
tight_layout()
show()<jupyter_output><empty_output><jupyter_text>Multiple normal sample distributions<jupyter_code>'''Show multiple samples from the same distribution, and compare means.'''
# Do this 25 times, and show the histograms
numRows = 5
numData = 100
for ii in range(numRows):
    for jj in range(numRows):
        data = stats.norm.rvs(myMean, mySD, size=numData)
        subplot(numRows,numRows,numRows*ii+jj+1)
        hist(data)

        xticks([])
        yticks([])
        xlim(myMean-3*mySD, myMean+3*mySD)

tight_layout()
show()

# Check out the mean of 1000 normally distributded samples
numTrials = 1000;
numData = 100
myMeans = ones(numTrials)*nan
for ii in range(numTrials):
    data = stats.norm.rvs(myMean, mySD, size=numData)
    myMeans[ii] = mean(data)
print('The standard error of the mean, with {0} samples, is {1:5.3f}'.format(numData, std(myMeans)))<jupyter_output><empty_output><jupyter_text>Normality Check<jupyter_code>'''Check if the distribution is normal.'''
# Generate and show a distribution
numData = 100
data = stats.norm.rvs(myMean, mySD, size=numData)
hist(data)
# Graphical test: if the data lie on a line, they are pretty much
# normally distributed
_ = stats.probplot(data, plot=plt)
# The scipy "normaltest" is based on D’Agostino and Pearson’s test that
# combines skew and kurtosis to produce an omnibus test of normality.
_, pVal = stats.normaltest(data)

# Or you can check for normality with Kolmogorov-Smirnov test: but this is only advisable for large sample numbers!
#_,pVal = stats.kstest((data-np.mean(data))/np.std(data,ddof=1), 'norm')

if pVal > 0.05:
    print('Data are probably normally distributed')<jupyter_output>Data are probably normally distributed<jupyter_text>Values from the Cumulative Distribution Function<jupyter_code>'''Calculate an empirical cumulative distribution function, compare it with the exact one, and
find the exact point for a specific data value.'''

# Generate normally distributed random data
myMean = 5
mySD = 2
numData = 100
data = stats.norm.rvs(myMean, mySD, size=numData)

# Calculate the cumulative distribution function, CDF
numbins = 20
counts, bin_edges = histogram(data, bins=numbins, normed=True)
cdf = cumsum(counts)
cdf /= max(cdf)

# compare with the exact CDF
step(bin_edges[1:],cdf)
plot(x, stats.norm.cdf(x, myMean, mySD),'r')

# Find out the value corresponding to the x-th percentile: the
# "cumulative distribution function"
value = 2
myMean = 5
mySD = 2
cdf = stats.norm.cdf(value, myMean, mySD)
print(('With a threshold of {0:4.2f}, you get {1}% of the data'.format(value, round(cdf*100))))

# For the percentile corresponding to a certain value: 
# the "inverse cumulative distribution function" 
value = 0.025
icdf = stats.norm.isf(value, myMean, mySD)
print('To get {0}% of the data, you need a threshold of {1:4.2f}.'.format((1-value)*100, icdf))<jupyter_output>With a threshold of 2.00, you get 7.0% of the data
To get 97.5% of the data, you need a threshold of 8.92.
{'fraction_non_alphanumeric': 0.11943269470017417, 'fraction_numerical': 0.029111719333167453, 'mean_word_length': 5.380952380952381, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '194019', 'license': 'Apache-2.0', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1367, 'n_words': 429, 'path': 'week4/day4/theory/matplotplib/1_distributions_matplotlib/4.distNormal.ipynb', 'repo_name': 'JavierOlcoz/data_science_nov_2020', 'size': '4019'}