<jupyter_start><jupyter_text>Normality Tests<jupyter_code>def Normality(data, name, ddof =1 , assume_discrete = False): 
    IQR = np.percentile(data, 75) - np.percentile(data, 25) 
    bins_width = 2 * IQR * (len(data)**(-1/3)) 
    suggest_bins = np.ceil((data.max() - data.min())/bins_width)
    
    if assume_discrete: 
        bins = arange(data.min(), data.max() + 2) - 0.5
        freq, bins, _ = plt.hist(data, bins = bins, normed = True , alpha=0.6) 
    else: 
        freq, bins, _ = plt.hist(data, bins= suggest_bins, normed = True, alpha = 0.6)
    X = st.norm(mean(data) , std(data, ddof = ddof))
    x1 = (bins[1:] + bins[:-1])/2
    y1 = X.pdf(x1) 
    plt.plot(x1,y1,color='red', marker = 'o')
    plt.title("Histogram")
    plt.xlabel(name) 
    plt.ylabel("Freq")
    plt.show() 
    
    x2 = sort(data) 
    probs = linspace(1, len(x2) , num = len(x2))/len(x2) 
    y2 = X.ppf(probs) 
    plt.plot(x2[:-1],y2[:-1],'o', color = 'blue') 
    plt.plot([x2.min() , x2.max()],[x2.min() , x2.max()],color= 'yellow',linewidth = 2) 
    plt.title("Line plot") 
    plt.xlabel(name) 
    plt.show() 
    
    print("Empirical Rule:") 
    x_bar, x_s = mean(data) , std(data, ddof = ddof) 
    for i in range(1,4): 
        within_std = np.sum(np.abs(data - x_bar) < i * x_s)
        print("Data within {} std: {:.2f}%".format(i, within_std/data.size * 100))
x  = df['effective_literacy_rate_total'].values
Normality(x,'effective_literacy_rate_total ')
y = df['sex_ratio'].values
Normality(y, 'sex Ratio')<jupyter_output>C:\Users\Gebruiker\AppData\Local\Continuum\Anaconda3\lib\site-packages\numpy\lib\function_base.py:564: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  n = np.zeros(bins, ntype)
C:\Users\Gebruiker\AppData\Local\Continuum\Anaconda3\lib\site-packages\numpy\lib\function_base.py:611: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  n += np.bincount(indices, weights=tmp_w, minlength=bins).astype(ntype)<jupyter_text>Linear Regression<jupyter_code>x  = df['effective_literacy_rate_total'].values
X = sm.add_constant(x)
#Use X for determining coefficients and predicted values



y = df['sex_ratio'].values
def fitted_model(X,y): 
    fitted_model = sm.OLS(y,X) 
    return fitted_model.fit() 

def coeff(X,y): 
    coeff = fitted_model(X,y).params
    return np.array(coeff)
    
def prediction(X,y): 
    n = len(X)
    predictions = fitted_model(X,y).predict(X) 
    return np.array(predictions.reshape(n,1))<jupyter_output><empty_output><jupyter_text>Linear regression line<jupyter_code>print("y= {:.3f} + {:.3f}x".format(coeff(X,y)[0] , coeff(X,y)[1]))<jupyter_output>y= 733.708 + 2.309x<jupyter_text>Prediction<jupyter_code>prediction(X,y)[:10]<jupyter_output><empty_output><jupyter_text>Correlation coefficient<jupyter_code>def cov(x,y,bias=0): 
    n = float(len(x))
    return np.sum((x - np.mean(x)) * (y - np.mean(y)))/(n - min(bias,1))

def z_score(x):
    return (x-np.mean(x))/np.std(x)

def correlation(x,y,bias = 0): 
    return cov(z_score(x),z_score(y),bias)
print("Correlation coefficient = {:.3f}".format(correlation(x,y)))<jupyter_output>Correlation coefficient = 0.256<jupyter_text>Determination Coefficient<jupyter_code>def R_squared(x,y):
    n = len(x)
    X = np.mat([np.ones(n), x]).T
    Y = np.mat(y).T
    M = X.T * X

    coef1 = M.I * (X.T * Y)

    YR = X * coef1
    Ygem = Y.mean()

    SSE = np.dot((Y - YR).T, Y - YR)
    SSY = np.dot((Y - Ygem).T, Y - Ygem)
    SST = np.dot((YR - Ygem).T, YR - Ygem)

    Rkwadraat = 1 - SSE / SSY
    return(Rkwadraat[0,0])
print("Determination coefficient = {:.4f}".format(R_squared(x,y)))<jupyter_output>Determination coefficient = 0.0654<jupyter_text>P-value<jupyter_code>def p_value(X,y):
    n = len(X) 
    R  = st.rdist(n - 2) 
    pvalue = 2*(1 - R.cdf(abs(correlation(x,y))))
    return pvalue
print("pvalue = {:.10f} ".format(p_value(X,y)))<jupyter_output>pvalue = 0.0000000084<jupyter_text>Regression plot<jupyter_code>def regression_plot(dataset,xname,yname,fitted_values): 
    x_range = [dataset[xname].min(), dataset[xname].max()]
    y_range = [dataset[yname].min(), dataset[yname].max()]
    
    scatter_plot = dataset.plot(kind='scatter', x = xname, y = yname, xlim=x_range,ylim=y_range, linewidth=10)
    
    y_hat = scatter_plot.plot(x_range,[dataset[yname].mean(), dataset[yname].mean()],'--', color='red', linewidth=1.5)
    x_hat = scatter_plot.plot([dataset[xname].mean(),dataset[xname].mean()],y_range, '--', color= 'red', linewidth=1.5) 
    
    regression_line = scatter_plot.plot(dataset[xname],fitted_values,color='yellow', linewidth=1.5)
    plt.title('Regression of {} and {}'.format(xname,yname))
    plt.show()
regression_plot(df,'effective_literacy_rate_total','sex_ratio',prediction(X,y) )
print("Correlation coefficient = {:.3f}".format(correlation(x,y)))
print("Coefficient of determination = {:.3f}".format(R_squared(x,y)))
print("pvalue = {:.10f} ".format(p_value(X,y)))<jupyter_output>Correlation coefficient = 0.256
Coefficient of determination = 0.065
pvalue = 0.0000000084
{'fraction_non_alphanumeric': 0.17733798991074892, 'fraction_numerical': 0.02619324796274738, 'mean_word_length': 4.22289766970618, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '283017', 'license': 'Apache-2.0', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2047, 'n_words': 335, 'path': 'Indian+Cities.ipynb', 'repo_name': 'jarmovanrooij/Jarmo', 'size': '5154'}