<jupyter_start><jupyter_text>Uygulamalı Veri Bilimi ve Makine Öğrenimi Eğitim Kampı  Kodluyoruz Ağustos-Eylül 2019 Ankara  2. hafta Geçtiğimiz hafta istatistik temellerine değindikAnahtar kelimeler:* Z Table* Normal dağılım (Gaussian) (çan)* Uniform dağılım* Poisson dağılımı* Probability Density Function (PDF)* Cumulative Distribution Function (CDF)  IRIS dataseti  Imports<jupyter_code>import statsmodels.stats.api as sms # İstatistik Kütüphanesi
import os # Operating system kütüphanesi, çalıştığımız directory'yi değiştiririz
import pandas as pd # Dataframe adında python'un özel tablo kütüphanesi
import seaborn as sns
from sklearn import datasets #Sckit-Learning kütüphanesindeki hazır 'toy' datasetler
import matplotlib.pyplot as plt # Plot etmek için bu şekilde çağırıyoruz
import numpy as np
#os.chdir(r"C:\Users\neutr\Desktop\kodluyoruz bootcamp") #directory değiştirebiliyoruz ama şimdilik gerek yok
iris = sns.load_dataset("iris") #hazır datasetini yüklüyoruz
iris.head() # verisetinin ilk 5 terimi ve headerların ismine bakıyoruz (pandas dataframe özelliği)
fig, ax = plt.subplots() #figür oluşturuyoruz
ax.scatter(iris['sepal_length'],iris['sepal_width']) # scatter plot yaptırarak yaprak uzunluğu ve genişliğini alıyoruz
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
colors={'setosa' : 'r', 'versicolor' : 'g', 'virginica' : 'b'}

fig, ax = plt.subplots() #Diğer figür 3 türe de hakim ancak tek renk var, çiçeklerin rengine göre plot edelim
for i in range(len(iris['sepal_length'])):
    ax.scatter(iris['sepal_length'][i],iris['sepal_width'][i],color=colors[iris['species'][i]])
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
fig, ax = plt.subplots()
for i in range(len(iris['petal_length'])):
    ax.scatter(iris['petal_length'][i],iris['petal_width'][i],color=colors[iris['species'][i]])
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
columns = iris.columns.drop(['species'])
x_data = range (0, iris.shape[0])
fig, ax = plt.subplots()
for column in columns:
    ax.plot(x_data, iris[column])
ax.set_title('Iris Dataset')
ax.legend(columns)

columns
plt.figure(figsize=(10,7))
plt.hist(iris['sepal_length'],bins=20, color='green')
plt.title("Sepal length in cm")
plt.xlabel("sepal length cm")
plt.ylabel("Count")
test_array=[3,5,7,8,9,10,11,13,124]
print("mean: " + str(np.mean(test_array)))
print("median: " + str(np.median(test_array)))
#median outlierlara karşı daha robust, mean outlier durumlarına karşı hassas
# Quetile bulmak:
# elimizdeki veriyi 4 çeyreğe bölüyoruz
test_array=[3,5,7,8,9,10,11,13,124,35,36,23,21,12,13,14,15,60,32,34,1,3,5,8,6,7]
print("1. Quertile %25: " + str(np.percentile(q=25, a=test_array))) 
print("2. Quertile %50: " + str(np.percentile(q=50, a=test_array))) 
print("3. Quertile %75: " + str(np.percentile(q=75, a=test_array))) 
print("4. Quertile %100: " + str(np.percentile(q=100, a=test_array)))
Q3=np.percentile(q=75, a=test_array)
Q1=np.percentile(q=25, a=test_array)
IQR=Q3-Q1
print("IQR (Inter Quertile Range) : " + str(IQR))
outlier=[]
outlier_min=Q1-1.5*IQR
outlier_max=Q3+1.5*IQR
for i in test_array:
    if i<outlier_min or i>outlier_max:
        outlier.append(i)
print("Bu sayıdan küçükse outlier diyebiliriz: " +str(outlier_min))
print("Bu sayıdan büyükse outlier diyebiliriz: " +str(outlier_max))
print(outlier)
plt.figure()
iris.boxplot()
f, axs = plt.subplots(2,2,figsize=(16,16))

ax = f.add_subplot(221)
plt.hist(iris['sepal_length'],bins=20, color='green')
plt.title("Sepal length in cm")
plt.xlabel("sepal length cm")
plt.ylabel("Count")

ax2 = f.add_subplot(222)
plt.hist(iris['sepal_width'],bins=20, color='blue')
plt.title("Sepal width in cm")
plt.xlabel("sepal width cm")
plt.ylabel("Count")

ax2 = f.add_subplot(223)
plt.hist(iris['petal_length'],bins=20, color='red')
plt.title("Petal length in cm")
plt.xlabel("Petal length cm")
plt.ylabel("Count")

ax2 = f.add_subplot(224)
plt.hist(iris['petal_width'],bins=20, color='yellow')
plt.title("Petal width in cm")
plt.xlabel("Petal width cm")
plt.ylabel("Count")
columns
iris_corr=iris.corr()
iris_corr
plt.matshow(iris_corr)
plt.show()
rs = np.random.RandomState(0)
df = pd.DataFrame(rs.rand(10,10))
corr = df.corr()
corr.style.background_gradient(cmap='coolwarm')<jupyter_output><empty_output><jupyter_text>Titanic Veriseti<jupyter_code>import os
os.chdir(r"C:\Users\neutr\Desktop\kodluyoruz bootcamp") #verisetini nereye indirdiyeseniz adresi giriniz
titanic=pd.read_csv('titanic_train.csv')
#data=pd.read_csv('https://raw.githubusercontent.com/caglarmert/UVBMOB/master/titanic_train.csv') # veriseti yoksa internetten çekmek için

titanic.head()
survived_gender=pd.crosstab(index=titanic['Survived'],columns=titanic['Sex'])
survived_gender.index = ["died","survived"]
survived_gender
survived_class=pd.crosstab(index=titanic['Survived'],columns=titanic['Pclass'])
survived_class.index = ["died","survived"]
survived_class
char_cabin = titanic["Cabin"].astype(str)
new_Cabin = np.array([cabin[0] for cabin in char_cabin])
titanic["Cabin"] = pd.Categorical(new_Cabin)
titanic["Cabin"]
survived_count=pd.crosstab(index=titanic['Survived'],columns=titanic['Cabin'])
survived_count.index = ["died","survived"]
survived_count
survived_class=pd.crosstab(index=titanic['Survived'],
                           columns=titanic['Pclass'],
                          margins=True)
survived_class.columns = ["Class1","Class2","Class3","Row_Total"]
survived_class.index = ["Died","Survived","Column_Total"]
survived_class
survived_class/survived_class.loc["Column_Total","Row_Total"]
d = {'Name':pd.Series(['Tom','James','Ricky','Vin','Steve','Smith','Jack',
   'Lee','David','Gasper','Betina','Andres']),
   'Age':pd.Series([25,26,25,23,30,29,23,34,40,30,51,46]),
   'Rating':pd.Series([4.23,3.24,3.98,2.56,3.20,4.6,3.8,3.78,2.98,4.80,4.10,3.65])
}
df=pd.DataFrame(d)

print(df.count())

print(df.sum())

print(df.mean())

print(df.median())

print(df.mode())

print(df.std())

print(df.min())

print(df.max())
print(df.describe(include='all'))
import seaborn as sns
from scipy import stats
from scipy.stats import norm
veri_normal = norm.rvs(size=10000,loc=0,scale=1)

ax = sns.distplot(veri_normal)
ax = sns.distplot(veri_normal, bins=20, kde=True, color= "orange")

ax.set(xlabel='Normal', ylabel='Frequency')
olcumler = np.array([17, 160, 234, 149, 145, 107, 197, 75, 201, 225, 211, 119, 
               157, 145, 127, 244, 163, 114, 145,  65, 112, 185, 202, 146,
               203, 224, 203, 114, 188, 156, 187, 154, 177, 95, 165, 50, 110, 
       216, 138, 151, 166, 135, 155, 84, 251, 173, 131, 207, 121, 120])
sns.distplot(olcumler)
ax = sns.distplot(olcumler)
ax = sns.distplot(olcumler, bins=6, kde=True, color= "orange")

ax.set(xlabel='Normal', ylabel='Frequency')
stats.describe(olcumler)
import pylab
stats.probplot(olcumler, dist='norm', plot=pylab)
pylab.show()
import statsmodels.api as sm

sm.qqplot(olcumler, line ='s')
from scipy.stats import shapiro
stat, p = shapiro(olcumler)

print('Statistics = %.3f, p = %.3f' % (stat,p))<jupyter_output>Statistics = 0.985, p = 0.785<jupyter_text>----------------------------<jupyter_code>len(olcumler)
stats.ttest_1samp(olcumler,popmean=170)

stats.t.ppf(q=0.025,df=len(olcumler)-1)

stats.t.cdf(x=-2.0095752344892093, df=len(olcumler)-1)
A = pd.DataFrame([30,27,21,27,29,30,20,20,27,32,35,22,24,23,25,27,23,27,23,25,21,18,24,26,33,26,27,28,19,25])
  
B = pd.DataFrame([37,39,31,31,34,38,30,36,29,28,38,28,37,37,30,32,31,31,27,32,33,33,33,31,32,33,26,32,33,29])
GRUP_A= np.arange(len(A))
GRUP_A = pd.DataFrame(GRUP_A)
GRUP_A[:]="A"
A = pd.concat([A, GRUP_A], axis =1)
A.columns=['GELIR', 'GRUP']
A.head()
GRUP_B = np.arange(len(B))
GRUP_B = pd.DataFrame(GRUP_B)
GRUP_B[:]="B"
B = pd.concat([B, GRUP_B], axis=1)
B.columns=['GELIR', 'GRUP']
B.head()
AB = pd.concat([A,B])
AB.columns=['GELIR', 'GRUP']
AB.head()
import seaborn as sns
sns.boxplot(x= "GRUP", y="GELIR",data=AB)
from scipy.stats import shapiro
shapiro(AB.GELIR)
shapiro(B.GELIR)
shapiro(A.GELIR)
from scipy.stats import levene
levene(A.iloc[:,0], B.iloc[:,0]) #equal variance test
from scipy import stats
stats.ttest_ind(A.iloc[:,0], B.iloc[:,0],equal_var=True)
#Bağımlı iki örneklem

oncesi = pd.DataFrame([123,119,119,116,123,123,121,120,117,118,121,121,123,119,121,118,124,121,125,
                       115,115,119,118,121,117,117,120,120,121,117,118,117,123,118,124,121,115,118,125,115])
sonrasi = pd.DataFrame([118,127,122,132,129,123,129,132,128,130,128,138,140,130,134,134,124,140,134
                        ,129,129,138,134,124,122,126,133,127,130,130,130,132,117,130,125,129,133,120,127,123])
birlikte= pd.concat([oncesi,sonrasi],axis=1)
birlikte.columns=["öncesi","sonrası"]
birlikte.head()
grup_oncesi=np.arange(len(oncesi))
grup_oncesi=pd.DataFrame(grup_oncesi)
grup_oncesi[:]="ONCESI"
A=pd.concat([oncesi,grup_oncesi],axis=1)
A.columns=["skor","grup"]

grup_sonrasi=np.arange(len(sonrasi))
grup_sonrasi=pd.DataFrame(grup_sonrasi)
grup_sonrasi[:]="SONRASI"
B=pd.concat([sonrasi,grup_sonrasi],axis=1)
B.columns=["skor","grup"]
A.head()
B.head()
AB= pd.concat([A,B])
AB.columns=["skor","grup"]
AB.head()
sns.boxplot(x= "grup", y="skor",data=AB)



levene(A.iloc[:,0], B.iloc[:,0]) #equal variance test



stats.ttest_ind(A.iloc[:,0], B.iloc[:,0],equal_var=True)
shapiro(A.iloc[:,0])
shapiro(B.iloc[:,0])
stat, p = shapiro(A.iloc[:,0])
print('Alanı Veren Nokta = %.3f, p = %.3f' % (stat,p))

#interpret

alpha=0.05
if p> alpha:
    print(" Örneklem Normal (Gaussian) dağılımdan gelmektedir (fail to Reject)")
else:
    print(" Örneklem Normal (Gaussian) dağılımdan gelmemektedir (Reject H0) ")
stats.ttest_rel(A.iloc[:,0], B.iloc[:,0])<jupyter_output><empty_output><jupyter_text>ANOVA<jupyter_code>A = pd.DataFrame([28,33,30,29,28,29,27,31,30,32,28,33,25,29,27,31,31,30,31,34,30,32,31,34,28,32,31,28,33,29])
B = pd.DataFrame([31,32,30,30,33,32,34,27,36,30,31,30,38,29,30,34,34,31,35,35,33,30,28,29,26,37,31,28,34,33])
C = pd.DataFrame([40,33,38,41,42,43,38,35,39,39,36,34,35,40,38,36,39,36,33,35,38,35,40,40,39,38,38,43,40,42])
dfs=[A,B,C]
ABC= pd.concat(dfs, axis =1)
ABC.columns = ["A","B","C"]
ABC.describe().T
ABC
for i in range(len(ABC.columns)):

    stat, p = shapiro(ABC.iloc[:,i])
    print(str(ABC.columns[i])+" için: ")
    print('Alanı Veren Nokta = %.3f, p = %.3f' % (stat,p))

    #interpret

    alpha=0.05
    if p> alpha:
        print(" Örneklem Normal (Gaussian) dağılımdan gelmektedir (fail to Reject)")
    else:
        print(" Örneklem Normal (Gaussian) dağılımdan gelmemektedir (Reject H0) ")
stat, p = levene(ABC.iloc[:,0],ABC.iloc[:,1],ABC.iloc[:,2])
print(str(ABC.columns[i])+" için: ")
print('Alanı Veren Nokta = %.3f, p = %.3f' % (stat,p))

#interpret

alpha=0.05
if p> alpha:
    print(" Örneklem Normal (Gaussian) dağılımdan gelmektedir (fail to Reject)")
else:
    print(" Örneklem Normal (Gaussian) dağılımdan gelmemektedir (Reject H0) ")

from scipy.stats import f_oneway
f_oneway(ABC['A'],ABC['B'],ABC['C'])<jupyter_output><empty_output><jupyter_text>IRIS dataset<jupyter_code>import statsmodels.stats.api as sms # İstatistik Kütüphanesi
import os # Operating system kütüphanesi, çalıştığımız directory'yi değiştiririz
import pandas as pd # Dataframe adında python'un özel tablo kütüphanesi
import seaborn as sns
from sklearn import datasets #Sckit-Learning kütüphanesindeki hazır 'toy' datasetler
import matplotlib.pyplot as plt # Plot etmek için bu şekilde çağırıyoruz
import numpy as np
#os.chdir(r"C:\Users\neutr\Desktop\kodluyoruz bootcamp") #directory değiştirebiliyoruz ama şimdilik gerek yok
iris = sns.load_dataset("iris") #hazır datasetini yüklüyoruz
iris.head() # verisetinin ilk 5 terimi ve headerların ismine bakıyoruz (pandas dataframe özelliği)
fig, ax = plt.subplots() #figür oluşturuyoruz
ax.scatter(iris['sepal_length'],iris['sepal_width']) # scatter plot yaptırarak yaprak uzunluğu ve genişliğini alıyoruz
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
colors={'setosa' : 'r', 'versicolor' : 'g', 'virginica' : 'b'}

fig, ax = plt.subplots() #Diğer figür 3 türe de hakim ancak tek renk var, çiçeklerin rengine göre plot edelim
for i in range(len(iris['sepal_length'])):
    ax.scatter(iris['sepal_length'][i],iris['sepal_width'][i],color=colors[iris['species'][i]])
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
fig, ax = plt.subplots()
for i in range(len(iris['petal_length'])):
    ax.scatter(iris['petal_length'][i],iris['petal_width'][i],color=colors[iris['species'][i]])
ax.set_title('Iris Dataset')
ax.set_xlabel('sepal length (cm)')
ax.set_ylabel('sepal width (cm)')
columns = iris.columns.drop(['species'])
x_data = range (0, iris.shape[0])
fig, ax = plt.subplots()
for column in columns:
    ax.plot(x_data, iris[column])
ax.set_title('Iris Dataset')
ax.legend(columns)

columns
plt.figure(figsize=(10,7))
plt.hist(iris['sepal_length'],bins=20, color='green')
plt.title("Sepal length in cm")
plt.xlabel("sepal length cm")
plt.ylabel("Count")
test_array=[3,5,7,8,9,10,11,13,124]
print("mean: " + str(np.mean(test_array)))
print("median: " + str(np.median(test_array)))
#median outlierlara karşı daha robust, mean outlier durumlarına karşı hassas
# Quetile bulmak:
# elimizdeki veriyi 4 çeyreğe bölüyoruz
test_array=[3,5,7,8,9,10,11,13,124,35,36,23,21,12,13,14,15,60,32,34,1,3,5,8,6,7]
print("1. Quertile %25: " + str(np.percentile(q=25, a=test_array))) 
print("2. Quertile %50: " + str(np.percentile(q=50, a=test_array))) 
print("3. Quertile %75: " + str(np.percentile(q=75, a=test_array))) 
print("4. Quertile %100: " + str(np.percentile(q=100, a=test_array)))
Q3=np.percentile(q=75, a=test_array)
Q1=np.percentile(q=25, a=test_array)
IQR=Q3-Q1
print("IQR (Inter Quertile Range) : " + str(IQR))
outlier=[]
outlier_min=Q1-1.5*IQR
outlier_max=Q3+1.5*IQR
for i in test_array:
    if i<outlier_min or i>outlier_max:
        outlier.append(i)
print("Bu sayıdan küçükse outlier diyebiliriz: " +str(outlier_min))
print("Bu sayıdan büyükse outlier diyebiliriz: " +str(outlier_max))
print(outlier)
plt.figure(figsize=(10,10))
iris.boxplot()
f, axs = plt.subplots(2,2,figsize=(16,16))

ax = f.add_subplot(221)
plt.hist(iris['sepal_length'],bins=20, color='green')
plt.title("Sepal length in cm")
plt.xlabel("sepal length cm")
plt.legend([iris.columns[0]])
plt.ylabel("Count")

ax2 = f.add_subplot(222)
plt.hist(iris['sepal_width'],bins=20, color='blue')
plt.title("Sepal width in cm")
plt.xlabel("sepal width cm")
plt.ylabel("Count")
plt.legend([iris.columns[1]])

ax2 = f.add_subplot(223)
plt.hist(iris['petal_length'],bins=20, color='red')
plt.title("Petal length in cm")
plt.xlabel("Petal length cm")
plt.ylabel("Count")
plt.legend([iris.columns[2]])

ax2 = f.add_subplot(224)
plt.hist(iris['petal_width'],bins=20, color='yellow')
plt.title("Petal width in cm")
plt.xlabel("Petal width cm")
plt.ylabel("Count")
plt.legend([iris.columns[3]])
columns
iris_corr=iris.corr()
iris_corr
plt.matshow(iris_corr)
plt.show()
rs = np.random.RandomState(0)
df = pd.DataFrame(rs.rand(10,10))
corr = df.corr()
corr.style.background_gradient(cmap='coolwarm')
grps=pd.unique(iris.species.values)
print(grps)

grps=pd.unique(iris.species.values)
for sepal_petal in iris.columns[:-1]:
    print (sepal_petal+" : ")

    for name in grps:
        print(name, "%.2f , %.2f" % shapiro(iris[sepal_petal][iris['species'] == name]))
    print (" ")
    iris.boxplot(column=sepal_petal, by="species",figsize=(6,6))
    plt.show()
    print (" ")
iris.head()
iris.columns[:-1]
from scipy.stats import bartlett
for i in iris.columns[:-1]:
    print(i)
    print("   ")
    
    print(bartlett(iris[i][iris['species']  == 'setosa'], 
                   iris[i][iris['species']  == 'versicolor'], 
                   iris[i][iris['species']  == 'virginica']))
    print("   ")
for i in iris.columns[:-1]:
    print(i)
    print("   ")
    
    print(levene(  iris[i][iris['species']  == 'setosa'], 
                   iris[i][iris['species']  == 'versicolor'], 
                   iris[i][iris['species']  == 'virginica']))
    print("   ")
for i in iris.columns[:-1]:
    print(i)
    print("   ")
    
    print(f_oneway(iris[i][iris['species']  == 'setosa'], 
                   iris[i][iris['species']  == 'versicolor'], 
                   iris[i][iris['species']  == 'virginica']))
    print("   ")<jupyter_output>sepal_length
   
F_onewayResult(statistic=119.26450218450468, pvalue=1.6696691907693826e-31)
   
sepal_width
   
F_onewayResult(statistic=49.160040089612075, pvalue=4.492017133309115e-17)
   
petal_length
   
F_onewayResult(statistic=1180.161182252981, pvalue=2.8567766109615584e-91)
   
petal_width
   
F_onewayResult(statistic=960.007146801809, pvalue=4.169445839443116e-85)
{'fraction_non_alphanumeric': 0.1832550953710856, 'fraction_numerical': 0.07522728623209936, 'mean_word_length': 6.170856412441415, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '637791', 'license': 'MIT', 'n_tokens_mistral': 8270, 'n_tokens_neox': 7257, 'n_words': 1188, 'path': 'Dersler/Hafta_2.ipynb', 'repo_name': 'UVBMOB/Ankara2019', 'size': '16829'}