<jupyter_start><jupyter_text>import pandas as pd<jupyter_code>data = pd.read_csv('WHO.csv')
data.head()
data.shape
data.isnull().sum()
#Need to treat 0s as there are no nans
data[data==0].count()
data['Region'].unique()
for i in data['Region'].unique():
    li=list(data[(data['FertilityRate'] ==0.0) & (data['Region'] ==i)].index)
    data['FertilityRate'][li]=data['FertilityRate'][data['Region']==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["CellularSubscribers"]==0.0)& data["Region"]==i].index)
    data['CellularSubscribers'][li]=data['CellularSubscribers'][data["Region"]==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["GNI"]==0.0)& data["Region"]==i].index)
    data['GNI'][li]=data['GNI'][data["Region"]==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["LiteracyRate"]==0.0)& data["Region"]==i].index)
    data['LiteracyRate'][li]=data['LiteracyRate'][data["Region"]==i].mean()
                   
#outliers
import matplotlib.pyplot as plt
plt.scatter(data['Population'].index, data['Population'])
from scipy import stats
import numpy as np
li=list(data['Population'][np.abs(stats.zscore(data["Population"]))>1].index)
data["Population"][li]=data.drop(li)['Population'].mean()
li
plt.scatter(data['Under15'].index,data['Under15'])
plt.scatter(data['Over60'].index,data['Over60'])
plt.scatter(data['GNI'].index , data['GNI'])
li=list(data['GNI'][np.abs(stats.zscore(data['GNI'])) > 3].index)
#capping
data['GNI']['li']=data['GNI'].sort_values().iloc[-5]
plt.scatter(data['CellularSubscribers'].index,data['CellularSubscribers'])
data.drop('Country',axis=1,inplace=True)
data.head()
sorted(data.Region.unique())
from sklearn import preprocessing
encoder = preprocessing.LabelEncoder()
for i in data.columns:
    if isinstance(data[i][0],str):
        data[i]=encoder.fit_transform(data[i])
data.head()
X=data.drop(['LifeExpectancy','PrimarySchoolEnrollmentMale' ,'PrimarySchoolEnrollmentFemale'],axis=1)
y=data['LifeExpectancy']
X.columns
from sklearn.model_selection import train_test_split
trainX ,testX , trainy , testy = train_test_split(X , y , test_size=0.5)
from sklearn.linear_model import LinearRegression
mlrm=LinearRegression()
mlrm.fit(trainX,trainy)
mlrm.score(trainX,trainy)*100
X.shape
y.shape
mlrm.score(testX,testy)*100
from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.tools.tools import add_constant
X_vif = add_constant(X)
pd.Series([variance_inflation_factor(X_vif.values,i) 
           for i in range (X_vif.shape[1])],
           index=X_vif.columns)
import scipy.stats as stats 
stats.f_oneway(X['Region'],y)
columns = list(X.columns)
columns.remove('Region')
print(columns)

correlation = pd.Series([np.abs(np.corrcoef(X[i],y)[0,1])
                        for i in columns], index = columns)
correlation
X.drop('Population',axis=1,inplace=True)
from sklearn.model_selection import train_test_split
trainX ,testX , trainy , testy = train_test_split(X , y , test_size=0.5)

from sklearn.linear_model import LinearRegression
mlrm=LinearRegression()
mlrm.fit(trainX,trainy)

print(mlrm.score(trainX,trainy)*100)
print(mlrm.score(testX,testy)*100)


import matplotlib.pyplot as plt
plt.figure(figsize=(16,8))
plt.plot(testy.reset_index(drop=True), label='Actual',color='g')
plt.plot(mlrm.predict(testX),label='Predicted',color='r')
plt.legend(loc='upper right')
#PCA starts so the whole thing is done again from the start
data=pd.read_csv('WHO.csv')
data.shape
encoder=preprocessing.LabelEncoder()

for i in data.columns:
    if isinstance (data[i][0],str):
        data[i]=encoder.fit_transform(data[i])
for i in data['Region'].unique():
    li=list(data[(data['FertilityRate'] ==0.0) & (data['Region'] ==i)].index)
    data['FertilityRate'][li]=data['FertilityRate'][data['Region']==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["CellularSubscribers"]==0.0)& data["Region"]==i].index)
    data['CellularSubscribers'][li]=data['CellularSubscribers'][data["Region"]==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["GNI"]==0.0)& data["Region"]==i].index)
    data['GNI'][li]=data['GNI'][data["Region"]==i].mean()
for i in data ['Region'].unique():
    li=list(data[(data["LiteracyRate"]==0.0)& data["Region"]==i].index)
    data['LiteracyRate'][li]=data['LiteracyRate'][data["Region"]==i].mean()
                   
li=list(data['GNI'][np.abs(stats.zscore(data['GNI'])) > 3].index)
data['GNI']['li']=data['GNI'].sort_values().iloc[-5]
sorted(data.Region.unique())
X=data.drop(['LifeExpectancy','PrimarySchoolEnrollmentMale' ,'PrimarySchoolEnrollmentFemale'],axis=1)
y=data['LifeExpectancy']
X.drop('Population',axis=1,inplace=True)
X=data.drop('LifeExpectancy',axis=1)
#MAIN CODE FOR PCA
from sklearn.preprocessing import StandardScaler 
  
scalar = StandardScaler() 
  
scalar.fit(X) 
scaled_data = scalar.transform(X) 

# Importing PCA 
from sklearn.decomposition import PCA 
  
# Let's say, components = 2 
pca = PCA() 
pca.fit(scaled_data) 


variance = pca.explained_variance_ratio_ #calculate variance ratios
variance

var=np.cumsum(np.round(pca.explained_variance_ratio_, decimals=3)*100)
print(var)
pca = PCA(n_components=8) 
X_transformed = pca.fit_transform(X) 

X_transformed.shape
from sklearn.model_selection import train_test_split
trainX ,testX , trainy , testy = train_test_split(X , y , test_size=0.2)

from sklearn.linear_model import LinearRegression
mlrm=LinearRegression()
mlrm.fit(trainX,trainy)

print(mlrm.score(trainX,trainy)*100)
print(mlrm.score(testX,testy)*100)
mlrm.coef_
data.dtypes<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18751111901796833, 'fraction_numerical': 0.013342821561999644, 'mean_word_length': 7.019971469329529, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '454418', 'license': 'MIT', 'n_tokens_mistral': 2092, 'n_tokens_neox': 2014, 'n_words': 312, 'path': 'life prac.ipynb', 'repo_name': 'sarthakwadhawan06/LIFE-EXPECTANCY', 'size': '5621'}