<jupyter_start><jupyter_text>BuildSys 2020 FiguresThe point of this notebook is to generate good quality figures for the BuildSys 2020 conference.<jupyter_code>import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Package Import<jupyter_code>import os
import sys
sys.path.append('../')

from src.features import build_features
from src.visualization import visualize
from src.reports import make_report

import pandas as pd
import numpy as np

from datetime import datetime, timedelta

import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib
import matplotlib.dates as mdates
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
import matplotlib.animation as animation<jupyter_output><empty_output><jupyter_text>Overview of FiguresThere are few figures we want to create for the poster. In general, we want to create four figures:1. Beacon2. Fitbit3. Beiwe4. Combination of the previous threeZoltan is also very adamant about animating our figures, so we need to look into that. He sent a [link](https://towardsdatascience.com/how-to-create-animated-graphs-in-python-bb619cc2dec1) that has some promising information.   Animation TestMore information on animation can be found using these blog posts:- [Somewhat Helpful](https://towardsdatascience.com/how-to-create-animated-graphs-in-python-bb619cc2dec1)- [More Helpful](https://towardsdatascience.com/animations-with-matplotlib-d96375c5442c)And documentation on the main animate function is [here](https://matplotlib.org/3.3.2/api/_as_gen/matplotlib.animation.FuncAnimation.html)<jupyter_code>beacon_data = pd.read_csv('../data/processed/bpeace2-beacon.csv',index_col=0,parse_dates=True)
# show it off:
beacon_data.head()
beacon_bb = beacon_data[beacon_data['Beacon'] == 19][datetime(2020,7,6):datetime(2020,7,14)]
beacon_bb = beacon_bb.resample('60T').mean()
beacon_bb_pollutants = beacon_bb[['CO2','PM_C_2p5','NO2','CO','TVOC','Lux','T_CO']]

normalized_df = (beacon_bb_pollutants-beacon_bb_pollutants.min())/(beacon_bb_pollutants.max()-beacon_bb_pollutants.min())
Writer = animation.writers['ffmpeg']
writer = Writer(fps=20, metadata=dict(artist='Me'), bitrate=1800)
#%matplotlib notebook
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib import animation

fig = plt.figure()
frames = 50
def init():
    initial = normalized_df.T
    initial = initial.replace(initial, 0)
    sns.heatmap(initial, vmax=1, cbar=False)

def animate(i):
    data = normalized_df.T[normalized_df.T < i/frames]
    sns.heatmap(data, vmax=1, cbar=False)

anim = animation.FuncAnimation(fig, animate, init_func=init, frames=frames, interval=1000, repeat = False)
anim.save('../test_heatmap.mp4', writer=writer)<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/seaborn/matrix.py:203: RuntimeWarning: All-NaN slice encountered
  vmin = np.nanmin(calc_data)<jupyter_text>Beacon      We are looking for a good summary of a week's worth of data. Looks:            week of July 6 is a good place to start        Beacon 19<jupyter_code>def create_cmap(colors,nodes):
    cmap = LinearSegmentedColormap.from_list("mycmap", list(zip(nodes, colors)))
    return cmap<jupyter_output><empty_output><jupyter_text>Static Plot<jupyter_code># Creating the dataframe
beacon_bb = beacon_data[beacon_data['Beacon'] == 19][datetime(2020,7,6):datetime(2020,7,14)]
beacon_bb = beacon_bb.resample('60T').mean()
beacon_bb_pollutants = beacon_bb[['CO2','PM_C_2p5','NO2','CO','TVOC','Lux','T_CO']]
fig, axes = plt.subplots(7,1,figsize=(16,7),sharex=True)
ylabels = ['CO$_2$',
           'PM$_{2.5}$',
           'NO$_2$',
           'CO',
           'TVOC',
           'Light',
           'T']
other_ylabels = ['ppm','$\mu$g/m$^3$','ppb','ppm','ppb','lux', '$^\circ$C']
cbar_ticks = [np.arange(400,1200,200),
          np.arange(0,40,10),
          np.arange(0,120,50),
          np.arange(0,15,4),
          [0,200,500,700],
          np.arange(0,250,50),
          np.arange(18,32,4)]
cmaps = [create_cmap(["green", "yellow", "orange", "red",],[0.0, 0.33, 0.66, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.2, 0.4, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.33, 0.66, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.375, 0.75, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.1, 0.31, 1]),
         create_cmap(["black","purple","red","orange","yellow","green"],[0.0, 0.1, 0.16, 0.2, 0.64, 1]),
         create_cmap(["cyan","blue","green","orange","red"],[0.0, 0.2, 0.4, 0.7, 1])]
for ax, var, low, high, ylabel, other_y, ticks, cmap in zip(axes,beacon_bb_pollutants.columns,[400,0,0,0,0,0,18],[1000,30,100,12,700,200,30],ylabels,other_ylabels,cbar_ticks,cmaps):
    sns.heatmap(beacon_bb_pollutants[[var]].T,vmin=low,vmax=high,ax=ax,cbar_kws={'ticks':ticks},cmap=cmap)
    ax.set_ylabel('')
    ax.text(0,-0.1,f'{ylabel} ({other_y})')
    ax.set_yticklabels([''])
    ax.set_xlabel('')

xlabels = ax.get_xticklabels()
new_xlabels = []
for label in xlabels:
    new_xlabels.append(label.get_text()[11:16])

ax.set_xticklabels(new_xlabels)


plt.subplots_adjust(hspace=0.6)
plt.savefig('../reports/BuildSys2020/beacon_heatmap.pdf',bbox_inches='tight')
plt.show()
plt.close()<jupyter_output><empty_output><jupyter_text>Animation<jupyter_code>#%matplotlib notebook
# Creating the dataframe
beacon_bb = beacon_data[beacon_data['Beacon'] == 19][datetime(2020,7,6):datetime(2020,7,14)]
beacon_bb = beacon_bb.resample('60T').mean()
beacon_bb_pollutants = beacon_bb[['CO2','PM_C_2p5','NO2','CO','TVOC','Lux','T_CO']]
# creating the figure
fig, axes = plt.subplots(7,1,figsize=(12,8),sharex=True)
ylabels = ['CO$_2$',
               'PM$_{2.5}$',
               'NO$_2$',
               'CO',
               'TVOC',
               'Light',
               'T']
other_ylabels = ['ppm','$\mu$g/m$^3$','ppb','ppm','ppb','lux', '$^\circ$C']
cbar_ticks = [np.arange(400,1200,200),
              np.arange(0,40,10),
              np.arange(0,120,50),
              np.arange(0,15,4),
              [0,200,500,700],
              np.arange(0,250,50),
              np.arange(18,32,4)]
cmaps = [create_cmap(["green", "yellow", "orange", "red",],[0.0, 0.33, 0.66, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.2, 0.4, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.33, 0.66, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.375, 0.75, 1]),
         create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.1, 0.31, 1]),
         create_cmap(["black","purple","red","orange","yellow","green"],[0.0, 0.1, 0.16, 0.2, 0.64, 1]),
         create_cmap(["cyan","blue","green","orange","red"],[0.0, 0.2, 0.4, 0.7, 1])]
for ax, var, low, high, ylabel, other_y, ticks, cmap in zip(axes,beacon_bb_pollutants.columns,[400,0,0,0,0,0,18],[1000,30,100,12,700,200,30],ylabels,other_ylabels,cbar_ticks,cmaps):
    sns.heatmap(beacon_bb_pollutants[[var]].T,vmin=low,vmax=high,ax=ax,cbar_kws={'ticks':ticks},cmap=cmap)
    ax.set_ylabel('')
    ax.text(0,-0.1,f'{ylabel} ({other_y})')
    ax.set_yticklabels([''])
    ax.set_xlabel('')
    
xlabels = ax.get_xticklabels()
new_xlabels = []
for label in xlabels:
    new_xlabels.append(label.get_text()[11:16])

ax.set_xticklabels(new_xlabels)
plt.subplots_adjust(hspace=0.6)

# Animation
frames = 193
Writer = animation.writers['ffmpeg']
writer = Writer(fps=20, metadata=dict(artist='Me'), bitrate=1800)
def init():
    initial = beacon_bb_pollutants
    initial = initial.replace(initial, 0)
    for ax, var in zip(axes,beacon_bb_pollutants.columns):
        sns.heatmap(initial[[var]].T, cbar=False, ax=ax)
    plt.subplots_adjust(hspace=0.6)
def animate(i):
    ylabels = ['CO$_2$',
               'PM$_{2.5}$',
               'NO$_2$',
               'CO',
               'TVOC',
               'Light',
               'T']
    other_ylabels = ['ppm','$\mu$g/m$^3$','ppb','ppm','ppb','lux', '$^\circ$C']
    cbar_ticks = [np.arange(400,1200,200),
              np.arange(0,40,10),
              np.arange(0,120,50),
              np.arange(0,15,4),
              [0,200,500,700],
              np.arange(0,250,50),
              np.arange(18,32,4)]
    cmaps = [create_cmap(["green", "yellow", "orange", "red",],[0.0, 0.33, 0.66, 1]),
             create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.2, 0.4, 1]),
             create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.33, 0.66, 1]),
             create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.375, 0.75, 1]),
             create_cmap(["green", "yellow", "orange", "red"],[0.0, 0.1, 0.31, 1]),
             create_cmap(["black","purple","red","orange","yellow","green"],[0.0, 0.1, 0.16, 0.2, 0.64, 1]),
             create_cmap(["cyan","blue","green","orange","red"],[0.0, 0.2, 0.4, 0.7, 1])]
    for ax, var, low, high, ylabel, other_y, ticks, cmap in zip(axes,beacon_bb_pollutants.columns,[400,0,0,0,0,0,18],[1000,30,100,12,700,200,30],ylabels,other_ylabels,cbar_ticks,cmaps):
        df = beacon_bb_pollutants[[var]].T
        #df = df[df <= np.nanmax(df)*((i+1)/frames)]
        df = df.iloc[:,:i+1]
        sns.heatmap(df,vmin=low,vmax=high,ax=ax,cbar=False,cmap=cmap)
        ax.set_ylabel('')
        ax.text(0,-0.1,f'{ylabel} ({other_y})')
        ax.set_yticklabels([''])
        ax.set_xlabel('')
        
    xlabels = ax.get_xticklabels()
    new_xlabels = []
    for label in xlabels:
        if label.get_text()[11:13] == '00':
            new_xlabels.append(label.get_text()[5:10] + ' ' + label.get_text()[11:16])
        else:
            new_xlabels.append(label.get_text()[11:16])
        
    ax.set_xticklabels(new_xlabels)
    
anim = animation.FuncAnimation(fig, animate, init_func=init, frames=frames, interval=500, repeat = False)
anim.save('../reports/BuildSys2020/beacon_heatmap.mp4', writer=writer)<jupyter_output><empty_output><jupyter_text>FitbitIn terms of purely Fitbit data, I was thinking of just showing something simple like daily steps with an overlay of the hourly numbers.<jupyter_code>fb_hourly = pd.read_csv('../data/processed/bpeace2-fitbit-intraday.csv',
                       index_col=0,parse_dates=True,infer_datetime_format=True)
fb_hourly.head()
fb_daily = pd.read_csv('../data/processed/bpeace2-fitbit-daily.csv',
                       index_col=0,parse_dates=True,infer_datetime_format=True)
fb_daily.head()<jupyter_output><empty_output><jupyter_text>The participant that corresponds to beacon 19 is qh34m4r9. We will continue to use this participant, at least initially to make sure all the data overlap.<jupyter_code>fb19_hourly = fb_hourly[fb_hourly['beiwe'] == 'qh34m4r9'][datetime(2020,7,6):datetime(2020,7,14)]
fb19_hourly = fb19_hourly.resample('60T').sum()
fb19_hourly.head()
fb19_daily = fb_daily[fb_daily['beiwe'] == 'qh34m4r9'][datetime(2020,7,6):datetime(2020,7,14)]
fb19_daily.head()<jupyter_output><empty_output><jupyter_text>Static Plot<jupyter_code>fig, ax1 = plt.subplots(figsize=(8,4.5))
ax2 = ax1.twinx() 

# Daily Steps - Bar
ax1.stem(fb19_daily.index, fb19_daily['activities_steps'], linefmt='k-',markerfmt='ko',basefmt='w ')
# Formatting the x-axis
ax1.xaxis.set_tick_params(rotation=-30)
ax1.set_xlim([datetime(2020,7,5,20),datetime(2020,7,14,4)])

# Formatting the y-axis (daily)
ax1.set_yticks([0,2000,4000,6000,8000])
ax1.set_ylabel('Daily Steps')

# Hourly Steps - Time Series
scatter_color = '#bf5700'
ax2.scatter(fb19_hourly.index.values, fb19_hourly['activities_steps'].values,marker='s',s=5,color=scatter_color)

# Formatting the y-axis (hourly)
ax2.set_yticks([0,1000,2000,3000,4000])
ax2.spines['right'].set_color(scatter_color)
ax2.yaxis.label.set_color(scatter_color)
ax2.tick_params(axis='y', colors=scatter_color)
ax2.set_ylabel('Hourly Steps')

# Formatting the x-axis (again)
ax2.xaxis.set_major_locator(mdates.DayLocator())
myFmt = mdates.DateFormatter('%m/%d')
ax2.xaxis.set_major_formatter(myFmt)

# Saving
plt.savefig('../reports/BuildSys2020/fitbit_steps.pdf')
plt.show()
plt.close()<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:5: UserWarning: In Matplotlib 3.3 individual lines on a stem plot will be added as a LineCollection instead of individual lines. This significantly improves the performance of a stem plot. To remove this warning and switch to the new behaviour, set the "use_line_collection" keyword argument to True.
  """<jupyter_text>BeiweThe best and simplest thing I can think to do with Beiwe is to include the submission times for the morning and weekly surveys.<jupyter_code>ema_morning = pd.read_csv('../data/processed/bpeace2-morning-survey.csv',
                         index_col=0,parse_dates=True,infer_datetime_format=True)
ema_morning.sort_index(inplace=True)
ema_morning.head()
ema_evening = pd.read_csv('../data/processed/bpeace2-evening-survey.csv',
                         index_col=0,parse_dates=True,infer_datetime_format=True)
ema_evening.sort_index(inplace=True)
ema_evening.head()
morning_b19 = ema_morning[ema_morning['ID'] == 'qh34m4r9'][datetime(2020,7,6):datetime(2020,7,14)]
morning_b19.head()
evening_b19 = ema_evening[ema_evening['ID'] == 'qh34m4r9'][datetime(2020,7,6):datetime(2020,7,14)]
evening_b19.head()<jupyter_output><empty_output><jupyter_text>Static Plot  First attemptNot well-received primarily because the way contentment is framed compared to the other moods.<jupyter_code>fig, ax = plt.subplots(figsize=(8,4.5))
# Plotting submission Times
for i in range(len(morning_b19)):
    if i == 0:
        ax.axvline(morning_b19.index[i],color='goldenrod',linestyle='dashed',zorder=1,label='Morning')
        ax.axvline(evening_b19.index[i],color='indigo',linestyle='dashed',zorder=2,label='Evening')
    else:
        ax.axvline(morning_b19.index[i],color='goldenrod',linestyle='dashed',zorder=1)
        ax.axvline(evening_b19.index[i],color='indigo',linestyle='dashed',zorder=2)
    
# Plotting mood scores
i = 0
for df in [morning_b19,evening_b19]:
    for mood, color, offset in zip(['Content','Stress','Lonely','Sad'],['seagreen','firebrick','gray','navy'],[270,90,-90,-270]):
        if i == 0:
            ax.scatter(df.index+timedelta(minutes=offset),df[mood].values,color=color,zorder=10,label=mood)
        else:
            ax.scatter(df.index+timedelta(minutes=offset),df[mood].values,color=color,zorder=10)
    i+= 1

# Formatting x-axis
ax.set_xlim([datetime(2020,7,6),datetime(2020,7,14,4)])
ax.xaxis.set_major_locator(mdates.DayLocator())
myFmt = mdates.DateFormatter('%m/%d')
ax.xaxis.set_major_formatter(myFmt)

# Formatting y-axis
ax.set_ylim([-0.5,3.5])
ax.set_yticks([0,1,2,3])
ax.set_yticklabels(['Not at all','A little bit','Quite a bit','Very much'])
ax.legend(loc='upper center',bbox_to_anchor=(1.1,1),ncol=1,frameon=False)

#plt.savefig('../reports/BuildSys2020/beiwe_mood.pdf')
plt.show()
plt.close()<jupyter_output><empty_output><jupyter_text>Second AttemptNow for the second attempt where we keep the mood on the same axis and color the points based on the response<jupyter_code>fig, ax = plt.subplots(figsize=(8,4.5))
# Plotting submission Times
for i in range(len(morning_b19)):
    if i == 0:
        ax.axvline(morning_b19.index[i],color='gray',linestyle='dashed',zorder=1,label='Morning')
        ax.axvline(evening_b19.index[i],color='black',linestyle='dashed',zorder=2,label='Evening')
    else:
        ax.axvline(morning_b19.index[i],color='gray',linestyle='dashed',zorder=1)
        ax.axvline(evening_b19.index[i],color='black',linestyle='dashed',zorder=2)
        
# Plotting mood scores
i = 0
for df in [morning_b19,evening_b19]:
    j = 0
    for mood, color in zip(['Content','Stress','Lonely','Sad'],['seagreen','firebrick','gray','navy']):
        ys = []
        shades = []
        for val in df[mood]:
            if val == 0:
                shades.append('red')
            elif val == 1:
                shades.append('orange')
            elif val == 2:
                shades.append('blue')
            elif val == 3:
                shades.append('green')
            else:
                shades.append('white')
                
            ys.append(j)
       
        if i == 0:
            ax.scatter(df.index,ys,color=shades,edgecolor='gray',marker='s',s=200,zorder=10)
        else:
            ax.scatter(df.index,ys,color=shades,edgecolor='black',marker='s',s=200,zorder=10)
            
        j += 1
    i+= 1

# Formatting x-axis
ax.set_xlim([datetime(2020,7,6),datetime(2020,7,14,4)])
ax.xaxis.set_major_locator(mdates.DayLocator())
myFmt = mdates.DateFormatter('%m/%d')
ax.xaxis.set_major_formatter(myFmt)

# Formatting y-axis
ax.set_ylim([-0.5,3.5])
ax.set_yticks([0,1,2,3])
ax.set_yticklabels(['Content','Stress','Lonely','Sad'])
ax.legend(loc='upper center',bbox_to_anchor=(1.1,1),ncol=1,frameon=False)

plt.savefig('../reports/BuildSys2020/beiwe_mood.pdf')
plt.show()
plt.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17596616938799484, 'fraction_numerical': 0.057147891460119817, 'mean_word_length': 4.420885068449539, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '512048', 'license': 'MIT', 'n_tokens_mistral': 7110, 'n_tokens_neox': 6427, 'n_words': 1012, 'path': 'notebooks/archive/0.3.0-hef-BuildSys20-figures.ipynb', 'repo_name': 'intelligent-environments-lab/utx000', 'size': '17026'}