<jupyter_start><jupyter_text>Model under development WARNINGFor plotly map rendering you must launch jupyter notebooks with a higher data limit: `jupyter notebook --NotebookApp.iopub_data_rate_limit=214748364` TODO - how to deal with missingness<jupyter_code>%load_ext autoreload
%autoreload 2
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from src.data.download_DHS import load_DHS_data
df = load_DHS_data()
df.shape
df.head()
df_wide = df.pivot(index='SurveyId', columns='IndicatorId', values='Value')
df_wide.head()
indicator_miss = df_wide.isna().mean().sort_values(ascending=False)
indicator_miss.head()
plt.plot(indicator_miss)
survey_miss = df_wide.isna().mean(axis=1).sort_values(ascending=False)
survey_miss.head()
plt.plot(survey_miss)<jupyter_output><empty_output><jupyter_text>Subset to latest country survey<jupyter_code>df.head()
latest_surveys = df.sort_values(['CountryName','SurveyYear'], ascending=False).drop_duplicates('CountryName')['SurveyId']
# df.sort_values(['CountryName', 'SurveyYear'], a).drop_duplicates()
latest_surveys.head()
df_latest = pd.merge(df, pd.DataFrame(latest_surveys))
print(len(df), len(df_latest))
df.sort_values(['CountryName', 'SurveyYear', 'IndicatorId']).head()
df_wide = df_latest.pivot(index='SurveyId', columns='IndicatorId', values='Value')
indicator_miss = df_wide.isna().mean().sort_values(ascending=False)
indicator_miss.head()
plt.plot(indicator_miss)
survey_miss = df_wide.isna().mean(axis=1).sort_values(ascending=False)
survey_miss.head()
plt.plot(survey_miss)
from sklearn.decomposition import PCA
from sklearn.impute import SimpleImputer
imputer = SimpleImputer()
n_components = 10
pca = PCA(n_components=n_components)
df_transformed = pd.DataFrame(imputer.fit_transform(df_wide), columns=df_wide.columns, index=df_wide.index)
cols = [f'component_{i}' for i in range(0, n_components)]
df_components = pd.DataFrame(pca.fit_transform(df_transformed), columns=cols, index=df_wide.index)
df_components.head()
df_all = pd.concat([df_components, df_transformed], axis=1)
df_all.head()
from src.data.download_DHS import IND_OTHER, IND_CHILD_RISKS
inds = IND_CHILD_RISKS
corr = df_all[cols + inds].corr()
plt.figure(figsize=[12,10])
g = sns.heatmap(corr,annot=False, fmt = ".2f", cmap = "coolwarm", center=0)
corr.head()
from src.data.download_DHS import INDICATORS
inds = [i for i in INDICATORS if i in df_all.columns]
corr = df_all[cols + inds].corr().drop(cols, axis=0).drop(inds, axis=1)
plt.figure(figsize=[10,20])
g = sns.heatmap(corr,annot=False, fmt = ".2f", cmap = "coolwarm", center=0)
df_components.head()
from sklearn.cluster import KMeans
kmean = KMeans(n_clusters=8)
kmean.fit(df_components)
df_all['cluster'] = kmean.labels_
df_transformed['cluster'] = kmean.labels_ # pd.Series(kmean.labels_, dtype='str')
# df_transformed['cluster'] = df_transformed['cluster'].astype(str)
df_transformed.cluster.value_counts()
df_transformed[['cluster']].head()
sns.scatterplot('CN_NUTS_C_WH3', 'FE_FRTR_W_GFR', hue='cluster', data=df_transformed)
sns.scatterplot('component_0', 'component_1', hue='cluster', data=df_all)
sns.catplot("cluster", y="FE_FRTR_W_GFR", data=df_all)
sns.catplot("cluster", y="component_0", data=df_all)
sns.scatterplot('component_0', 'FE_FRTR_W_GFR', hue='cluster', data=df_all)
inds = [i for i in INDICATORS if i in df_all.columns]
data = df_all.groupby('cluster')[inds].mean().T
data.head()
# plt.figure(figsize=[10,20])
# g = sns.heatmap(data,annot=False, fmt = ".2f", cmap = "coolwarm", center=0)
from sklearn.preprocessing import StandardScaler, MinMaxScaler
scaler = MinMaxScaler() # StandardScaler()
data = pd.DataFrame(scaler.fit_transform(df_all.drop('cluster', axis=1)), 
             columns=df_all.drop('cluster', axis=1).columns, index=df_all.index)
data['cluster'] = df_all['cluster']
data.head()
data = data.groupby('cluster')[inds].mean().T
data.head()
StandardScaler?
plt.figure(figsize=[10,20])
g = sns.heatmap(data,annot=False, fmt = ".2f", cmap = "coolwarm", center=0.5)<jupyter_output><empty_output><jupyter_text>Map clusters<jupyter_code>from src.data.download_DHS import load_country_codebook
codebook = load_country_codebook()
dhs_country_codebook = pd.merge(codebook[['CountryName', 'ISO3_CountryCode']], df.drop_duplicates('SurveyId')[['SurveyId', 'CountryName']])
dhs_country_codebook.head()
df_clusters = pd.merge(dhs_country_codebook, df_transformed.reset_index())
df_clusters.head()
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
# Connect Javascript to display the plots in the notebook
init_notebook_mode(connected=True)
# # Standard plotly imports
# import plotly.plotly as py
# import plotly.graph_objs as go
# from plotly.offline import iplot
# # Using cufflinks in offline mode
# import cufflinks
# cufflinks.go_offline()
scale = [[0, 'rgb(166,206,227)'], 
         [1, 'rgb(31,120,180)'], 
         [2, 'rgb(178,223,138)'], 
         [3, 'rgb(51,160,44)'], 
         [4, 'rgb(251,154,153)'],
         [5, 'rgb(251,154,153)'],
         [6, 'rgb(251,154,153)'],
         [7, 'rgb(227,26,28)']]
data = [dict(type='choropleth',
           locations=df_clusters['ISO3_CountryCode'], 
           z = df_clusters['cluster'],
            text=df_clusters['CountryName'],
             colorscale='Blackbody', #scale, Rainbow
             autocolorscale = False,
            colorbar = {'title': 'Indicator'}
            )]
layout = dict(title='My Title',
             geo=dict(showframe=False,
                     projection={'type':'natural earth'}))
fig = dict(data=data, layout=layout)
iplot(fig)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14990277532260915, 'fraction_numerical': 0.022626834010959874, 'mean_word_length': 6.425196850393701, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '507229', 'license': 'MIT', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2033, 'n_words': 365, 'path': 'notebooks/_archive/2019_01_25_first_clustering_model.ipynb', 'repo_name': 'zwrankin/health_indicators', 'size': '5657'}