<jupyter_start><jupyter_text>Parallel Coordinates plotSeeing if it's possible to replace the comparison scatterplot with a parallel coordinates plot<jupyter_code>%load_ext autoreload
%autoreload 2
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from src.data.utils import DATA_DIR
from src.visualization.utils import palette, get_palette
df = pd.read_csv(f'{DATA_DIR}/processed/GBD_child_health_indicators.csv')
location_metadata = pd.read_csv(f'{DATA_DIR}/metadata/gbd_location_metadata.csv')
indicators = list(df.indicator.unique())
n_neighbors = 4
year_ids = list(df.year_id.unique())

# Indicator Value by country in wide format
index_vars = ['location_name', 'year_id']
df_wide = df.pivot_table(index=index_vars, columns='indicator', values='val').reset_index()
df_wide = pd.merge(location_metadata, df_wide)
df_wide.head()
location_name = 'Nigeria'
countries = []
comparison_type = 'Countries'
year = 2017
data = df_wide.query(f'year_id == {year}')[['location_name'] + indicators].set_index('location_name')
l_data = data.loc[location_name]
similarity = np.abs(data ** 2 - l_data ** 2).sum(axis=1).sort_values()
locs = similarity[:n_neighbors + 1].index.tolist()
if countries is not None:
    locs += countries
df_similar = data.loc[locs]
if comparison_type == 'Value':
    title = f'Indicators of {location_name} and similar countries'
elif comparison_type == 'Comparison':
    df_similar = (df_similar - l_data)
    title = f'Indicators of countries relative to {location_name}'
df_similar = df_similar.reset_index().melt(id_vars='location_name', var_name='indicator')
df_similar.sort_values(['location_name', 'indicator'], ascending=[True, False], inplace=True)
df_similar['size'] = 10
df_similar.loc[df_similar.location_name == location_name, 'size'] = 14
df_similar.head()
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
# Connect Javascript to display the plots in the notebook
init_notebook_mode(connected=True)<jupyter_output><empty_output><jupyter_text>Existing Plot![image.png](attachment:image.png)<jupyter_code>plot = [go.Scatter(
            x=df_similar[df_similar['location_name'] == i]['value'],
            y=df_similar[df_similar['location_name'] == i]['indicator'],
            text=str(i),
            mode='markers',
            opacity=0.7,
            marker={
                'size': df_similar[df_similar['location_name'] == i]['size'],
                'line': {'width': 0.5, 'color': 'white'}
            },
            name=str(i)
        ) for i in df_similar.location_name.unique()]

iplot({
        'data': plot,
        'layout': go.Layout(
            title=f'Indicators of {location_name} and similar countries',
            height=150 + 20 * len(indicators),
            margin={'l': 200, 'b': 30, 't': 30, 'r': 0},
            hovermode='closest'
        )})
df[df.indicator == 'Tuberculosis']['value']<jupyter_output><empty_output><jupyter_text>Parallel coordinates plot<jupyter_code>df.indicator.unique().tolist().remove('Under-5 Mortality Rate')
df.indicator.unique().tolist()
from src.visualization.utils import palette
#colorscale = [[0,'#D7C16B'],[0.5,'#23D8C3'],[1,'#F3F10F']]
df = df_similar.copy()
df['cluster'] = np.random.randint(0, 5, len(df)) # HACK - initialize clusters
colorscale = 
data = [
    go.Parcoords(
        line = dict(color = df['cluster'], colorscale=palette), #colorscale = [['Nigeria','#D7C16B']], showscale = True
        dimensions = list(
            [dict(
                range = [0,100], constraintrange = [0,100],
                label = 'U5MR', values = df[df.indicator == 'Under-5 Mortality Rate']['value'])] + 
            [dict(
                range = [0,100],
                label = i, values = df[df.indicator == i]['value'])
        for i in df.indicator.unique()]
        )
    )]

layout = go.Layout(
    plot_bgcolor = '#E5E5E5',
    paper_bgcolor = '#E5E5E5'
)

iplot(data, layout)
def cluster_kmeans(n_clusters, indicators, year):
    df_c = df_wide.query(f'year_id == {year}')[['location_name'] + indicators].set_index('location_name')
    kmean = KMeans(n_clusters=n_clusters, random_state=0)
    kmean.fit(df_c)

    # Get U5MR by cluster and map cluster number to rank
    df_ordered = df_wide.query(f'year_id == {year}')
    df_ordered['cluster'] = kmean.labels_
    df_ordered = df_ordered.groupby('cluster')['Under-5 Mortality Rate'].mean().reset_index()
    df_ordered['U5MR_rank'] = df_ordered['Under-5 Mortality Rate'].rank().astype(
        'int') - 1  # rank starts at 1, we want 0-indexed
    cluster_map = df_ordered.set_index('cluster')['U5MR_rank'].to_dict()

    # Set cluster equal to U5MR rank
    df_c.reset_index(inplace=True)
    df_c['cluster'] = pd.Series(kmean.labels_).map(cluster_map)
    df_c = pd.merge(location_metadata, df_c)
    df_c['color'] = df_c.cluster.map(get_palette(n_clusters))
    return df_c.to_json()
indicators = df.indicator.unique().tolist()
df_c = pd.read_json(cluster_kmeans(5, indicators, 2017))
df_c.head()
df_c.cluster.unique()
n_clusters = len(df_c.cluster.unique())
colorscale = list(zip(palette.keys(), [palette[k] for k in palette.keys()]))
colorscale = [[0,'#0000ff'],[1,'#00bfff'],[2,'#00ffbf'], [3,'#40ff00'],[4,'#f2f20d']]
colorscale = [[k / (n_clusters - 1), get_palette(n_clusters)[k]] for k in
                  range(0, n_clusters)]  # choropleth colorscale seems to need 0-1 range
colorscale
data = [
    go.Parcoords(
        # test=None,
        line = dict(color = df_c['cluster'], colorscale=colorscale), #colorscale = [['Nigeria','#D7C16B']], showscale = True
        # opacity=0.1,
        # name=['test'] + indicators,
        # hoverinfo='test',
        dimensions = list(
            [dict(
                range = [0,100], constraintrange = [0,100],
                label = 'U5MR', values = df_c['Under-5 Mortality Rate'])] + 
            [dict(
                range = [0,100],
                tickvals=[100],
                tickformat=dict(visible=True, tickangle=180),
                label = i, 
                values = df_c[i])
        for i in indicators]
        )
    )]

# layout = go.Layout(
#     xaxis = dict(visible=False, range=(0,10), title='TEST')
# )

layout = go.Layout(
    autosize=False,
    # width=1600,
    height=80,
    plot_bgcolor = '#E5E5E5',
    paper_bgcolor = '#E5E5E5',

    title='Charity Comparison Plot'
)

iplot(data, layout)
df.species_id.unique()
df = pd.read_csv("https://raw.githubusercontent.com/bcdunbar/datasets/master/iris.csv")

data = [
    go.Parcoords(
        line = dict(color = df['species_id'],
                   colorscale = [[0,'#D7C16B'],[0.5,'#23D8C3'],[1,'#F3F10F']]),
        dimensions = list([
            dict(range = [0,8],
                constraintrange = [4,8],
                label = 'Sepal Length', values = df['sepal_length']),
            dict(range = [0,8],
                label = 'Sepal Width', values = df['sepal_width']),
            dict(range = [0,8],
                label = 'Petal Length', values = df['petal_length']),
            dict(range = [0,8],
                label = 'Petal Width', values = df['petal_width'])
        ])
    )
]

layout = go.Layout(
    plot_bgcolor = '#E5E5E5',
    paper_bgcolor = '#E5E5E5'
)

iplot(data, layout)
data = [
    go.Parcoords(
        line = dict(color = 'blue'),
        dimensions = list([
            dict(range = [1,5],
                 constraintrange = [1,2],
                 label = 'A', values = [1,4]),
            dict(range = [1.5,5],
                 tickvals = [1.5,3,4.5],
                 label = 'B', values = [3,1.5]),
            dict(range = [1,5],
                 tickvals = [1,2,4,5],
                 label = 'C', values = [2,4],
                 ticktext = ['text 1', 'text 2', 'text 3', 'text 4']),
            dict(range = [1,5],
                 label = 'D', values = [4,2])
        ])
    )
]

iplot(data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1569439217162213, 'fraction_numerical': 0.026721866767030485, 'mean_word_length': 3.0694231750893315, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '518367', 'license': 'MIT', 'n_tokens_mistral': 2888, 'n_tokens_neox': 2716, 'n_words': 533, 'path': 'notebooks/_archive/2019_02_14_parallel_coordinates_plot.ipynb', 'repo_name': 'zwrankin/health_indicators', 'size': '7971'}