<jupyter_start><jupyter_text>Data Analysis - Data Exploration<jupyter_code>import pandas as pd
import sklearn
import missingno as msno
import numpy as np
from sklearn.impute import KNNImputer
import sklearn.neighbors._base
import sys
sys.modules['sklearn.neighbors.base'] = sklearn.neighbors._base

from sklearn.decomposition import PCA
from missingpy import MissForest
from sklearn.cluster import KMeans, AgglomerativeClustering
import matplotlib.pyplot as plt
from yellowbrick.cluster import SilhouetteVisualizer
import warnings
warnings.filterwarnings('ignore')
NUTS2_GDP_EDU = pd.read_pickle("NUTS2.pkl")
pd.set_option('display.max_rows', 100)
NUTS2_GDP_EDU<jupyter_output><empty_output><jupyter_text>Changing EDU data type to float<jupyter_code>#ED8 is the raw number of Doctoral students 
NUTS2_GDP_EDU["EDU_2019_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2019_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2018_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2018_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2017_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2017_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2016_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2016_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2015_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2015_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2014_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2014_ED8"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2013_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2013_ED8"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2012_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED8"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED8"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED8"], downcast="float",errors='coerce')
#ED7 is the raw number of Master students 
NUTS2_GDP_EDU["EDU_2019_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2019_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2018_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2018_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2017_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2017_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2016_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2016_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2015_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2015_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2014_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2014_ED7"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2013_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2013_ED7"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2012_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED7"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED7"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED7"], downcast="float",errors='coerce')
#ED6 is the raw number of Bachelor students 
NUTS2_GDP_EDU["EDU_2019_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2019_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2018_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2018_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2017_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2017_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2016_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2016_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2015_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2015_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2014_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2014_ED6"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["EDU_2013_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2013_ED6"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2012_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED6"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED6"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED6"], downcast="float",errors='coerce')
msno.matrix(NUTS2_GDP_EDU)
msno.bar(NUTS2_GDP_EDU)<jupyter_output><empty_output><jupyter_text>Changing GDP data type to float<jupyter_code>#gdp_m represents GDP in millions of euros in a purticular region
NUTS2_GDP_EDU["GDP_2019_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2019_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2018_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2018_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2017_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2017_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2016_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2016_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2015_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2015_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2014_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2014_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2013_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2013_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2012_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2012_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2011_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2011_M"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2010_M"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2010_M"], downcast="float",errors='coerce')
#gdp euros per induvidual in a purticular NUTS region
NUTS2_GDP_EDU["GDP_2019"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2019"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2018"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2018"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2017"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2017"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2016"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2016"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2015"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2015"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2014"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2014"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2013"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2013"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2012"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2012"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2011"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2011"], downcast="float",errors='coerce')
NUTS2_GDP_EDU["GDP_2010"] = pd.to_numeric(NUTS2_GDP_EDU["GDP_2010"], downcast="float",errors='coerce')
msno.heatmap(NUTS2_GDP_EDU)<jupyter_output><empty_output><jupyter_text>A value near -1 means if one variable appears then the other variable is very likely to be missing.A value near 0 means there is no dependence between the occurrence of missing values of two variables.A value near 1 means if one variable appears then the other variable is very likely to be present  Miss Forrest missing imputation    Advantages:1)Can be applied to mixed data types (missings in numeric & categorical variables)2)No pre-processing required (no dummy-coding, standardization, data splitting, etc.)3)No assumptions required (aside from the normal assumption of being MAR/MCAR)4)Robust to noisy data, as random forests effectively have build-in feature selection. Methods like KNN imputation will have poor predictions in datasets with weak & non-informative predictors, whereas missForest() will make little to no use of these features5)Non-parametric: makes no assumptions about the relationship between the features, unlike MICE which assumes linearityExcellent predictive power6)Can leverage non-linear and interaction effects between features to improve imputation accuracy7)Gives an OOB error estimate for its predictions (Numeric: NRMSE/MSE, Categorical: PFC)8)Works with high dimensionality data (p≫n)Disadvantages:1)Imputation time, which increases with the number of observations, predictors and number of predictors containing missing values2)It inherits the same lack of interpretability of random forests3)It is an algorithm, not a model object you can store somewhere. This means it has to run each time missing data has to be imputed, which could be problematic in some production environments<jupyter_code>NaN_d = NUTS2_GDP_EDU.drop(['FREQ','GEO'],axis = 1)
imputer = MissForest()
X_imputed = imputer.fit_transform(NaN_d)
Filled = pd.DataFrame(X_imputed)
Filled
NUTS2_GDP_EDU_I = pd.DataFrame()

NUTS2_GDP_EDU_I['FREQ'] = NUTS2_GDP_EDU['FREQ']
NUTS2_GDP_EDU_I['GEO'] = NUTS2_GDP_EDU['GEO']
#ED8 is the raw number of Doctoral students 
NUTS2_GDP_EDU_I["EDU_2019_ED8"] = Filled.loc[:,0]
NUTS2_GDP_EDU_I["EDU_2018_ED8"] = Filled.loc[:,1]
NUTS2_GDP_EDU_I["EDU_2017_ED8"] = Filled.loc[:,2]
NUTS2_GDP_EDU_I["EDU_2016_ED8"] = Filled.loc[:,3]
NUTS2_GDP_EDU_I["EDU_2015_ED8"] = Filled.loc[:,4]
NUTS2_GDP_EDU_I["EDU_2014_ED8"] = Filled.loc[:,5]
NUTS2_GDP_EDU_I["EDU_2013_ED8"] = Filled.loc[:,6]
#NUTS2_GDP_EDU["EDU_2012_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED8"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED8"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED8"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED8"], downcast="float",errors='coerce')
#ED7 is the raw number of Master students 
NUTS2_GDP_EDU_I["EDU_2019_ED7"] = Filled.loc[:,7]
NUTS2_GDP_EDU_I["EDU_2018_ED7"] = Filled.loc[:,8]
NUTS2_GDP_EDU_I["EDU_2017_ED7"] = Filled.loc[:,9]
NUTS2_GDP_EDU_I["EDU_2016_ED7"] = Filled.loc[:,10]
NUTS2_GDP_EDU_I["EDU_2015_ED7"] = Filled.loc[:,11]
NUTS2_GDP_EDU_I["EDU_2014_ED7"] = Filled.loc[:,12]
NUTS2_GDP_EDU_I["EDU_2013_ED7"] = Filled.loc[:,13]
#NUTS2_GDP_EDU["EDU_2012_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED7"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED7"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED7"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED7"], downcast="float",errors='coerce')
#ED6 is the raw number of Bachelor students 
NUTS2_GDP_EDU_I["EDU_2019_ED6"] = Filled.loc[:,14]
NUTS2_GDP_EDU_I["EDU_2018_ED6"] = Filled.loc[:,15]
NUTS2_GDP_EDU_I["EDU_2017_ED6"] = Filled.loc[:,16]
NUTS2_GDP_EDU_I["EDU_2016_ED6"] = Filled.loc[:,17]
NUTS2_GDP_EDU_I["EDU_2015_ED6"] = Filled.loc[:,18]
NUTS2_GDP_EDU_I["EDU_2014_ED6"] = Filled.loc[:,19]
NUTS2_GDP_EDU_I["EDU_2013_ED6"] = Filled.loc[:,20]
#NUTS2_GDP_EDU["EDU_2012_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2012_ED6"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2011_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2011_ED6"], downcast="float",errors='coerce')
#NUTS2_GDP_EDU["EDU_2010_ED6"] = pd.to_numeric(NUTS2_GDP_EDU["EDU_2010_ED6"], downcast="float",errors='coerce')
#gdp_m represents GDP in millions of euros in a purticular region
NUTS2_GDP_EDU_I["GDP_2019_M"] = Filled.loc[:,21]
NUTS2_GDP_EDU_I["GDP_2018_M"] = Filled.loc[:,22]
NUTS2_GDP_EDU_I["GDP_2017_M"] = Filled.loc[:,23]
NUTS2_GDP_EDU_I["GDP_2016_M"] = Filled.loc[:,24]
NUTS2_GDP_EDU_I["GDP_2015_M"] = Filled.loc[:,25]
NUTS2_GDP_EDU_I["GDP_2014_M"] = Filled.loc[:,26]
NUTS2_GDP_EDU_I["GDP_2013_M"] = Filled.loc[:,27]
NUTS2_GDP_EDU_I["GDP_2012_M"] = Filled.loc[:,28]
NUTS2_GDP_EDU_I["GDP_2011_M"] = Filled.loc[:,29]
NUTS2_GDP_EDU_I["GDP_2010_M"] = Filled.loc[:,30]
#gdp euros per induvidual in a purticular NUTS region
NUTS2_GDP_EDU_I["GDP_2019"] = Filled.loc[:,31]
NUTS2_GDP_EDU_I["GDP_2018"] = Filled.loc[:,32]
NUTS2_GDP_EDU_I["GDP_2017"] = Filled.loc[:,33]
NUTS2_GDP_EDU_I["GDP_2016"] = Filled.loc[:,34]
NUTS2_GDP_EDU_I["GDP_2015"] = Filled.loc[:,35]
NUTS2_GDP_EDU_I["GDP_2014"] = Filled.loc[:,36]
NUTS2_GDP_EDU_I["GDP_2013"] = Filled.loc[:,37]
NUTS2_GDP_EDU_I["GDP_2012"] = Filled.loc[:,38]
NUTS2_GDP_EDU_I["GDP_2011"] = Filled.loc[:,39]
NUTS2_GDP_EDU_I["GDP_2010"] = Filled.loc[:,40]
msno.matrix(NUTS2_GDP_EDU_I)<jupyter_output><empty_output><jupyter_text>K Means Clustering   1) Standarization of the variables is not required 2) Every run is bound to give slightly different result as the starting point of the K-Means is random 3) The siloehts of the K-means clusters will be analyzed to give further insignt   Elbow Method<jupyter_code>K_mean = pd.DataFrame(NUTS2_GDP_EDU_I)
K_mean =  K_mean.drop(columns=['FREQ'])
K_means_D = K_mean.drop(columns=['GEO'])
K_means_D.head()
K_mean
wcss = []
for i in range(1, 15):
    kmeans = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)
    kmeans.fit(K_means_D)
    wcss.append(kmeans.inertia_)
print(wcss)
plt.plot(range(1, 15), wcss)
plt.title('Elbow Method')
plt.xlabel('Number of clusters')
plt.ylabel('WCSS')
plt.show()
pca = PCA(3)
 #Transform the data
pca_d = pca.fit_transform(K_means_D)
pca_d.shape
pca.explained_variance_ratio_<jupyter_output><empty_output><jupyter_text>From the graph above we can safely take upto 3 clusters in out data and even 2 clusters which has a good WSS score can be taken<jupyter_code>kmeans = KMeans(n_clusters= 3)
np.random.seed(1234)

k5cls = kmeans.fit(pca_d)
k5cls.labels_
#filter rows of original data
filtered_label0 = pca_d[k5cls.labels_ == 0]
filtered_label1 = pca_d[k5cls.labels_ == 1]
filtered_label2 = pca_d[k5cls.labels_ == 2]
 
#Plotting the results
plt.scatter(filtered_label0[:,0] , filtered_label0[:,1] , color = 'red')
plt.scatter(filtered_label1[:,0] , filtered_label1[:,1] , color = 'black')
plt.scatter(filtered_label2[:,0] , filtered_label2[:,1] , color = 'black')
plt.show()
kmeans = KMeans(n_clusters= 4)
np.random.seed(1234)

k5cls = kmeans.fit(K_means_D)
k5cls.labels_
filtered_label0 = K_mean[k5cls.labels_ == 0]
filtered_label0
filtered_label1 = K_mean[k5cls.labels_ == 1]
filtered_label1
filtered_label2 = K_mean[k5cls.labels_ == 2]
filtered_label2
filtered_label3 = K_mean[k5cls.labels_ == 3]
filtered_label3
f, ax = plt.subplots(1, figsize=(15, 15), dpi =50)
visualizer = SilhouetteVisualizer(kmeans, colors='yellowbrick')
visualizer.fit(K_means_D)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14731951393852752, 'fraction_numerical': 0.07719799857040743, 'mean_word_length': 10.668890742285237, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '144112', 'license': 'MIT', 'n_tokens_mistral': 6988, 'n_tokens_neox': 6335, 'n_words': 836, 'path': 'Data_Analysis .ipynb', 'repo_name': 'venkateshIedge7/CDC_Assignment1', 'size': '13990'}