<jupyter_start><jupyter_text>Table of Contents* [Intro](Intro)* [Load Data](Load-Data)* [Generate and Save Stats](Generate-and-Save-Stats)* [Basic Stats](Basic-Stats)	* [Preliminary Stats](Preliminary-Stats)	* [Weekday Stats](Weekday-Stats)	* [Monthly Stats](Monthly-Stats)	* [Year-Month Stats](Year-Month-Stats)	* [Weekday Stats By Month](Weekday-Stats-By-Month)	* [Daily Stats](Daily-Stats)* [Intraday Stats](Intraday-Stats)	* [Daily Stats](Daily-Stats)  Intro This notebook explores analysis and visualization of Fitbit sleep data.<jupyter_code>import os
import sys
from os.path import join

import pandas as pd
import numpy as np
import seaborn as sns
import time
import datetime
import matplotlib.pyplot as plt

sys.path.append(os.path.join(os.getcwd(), "src"))
from resources import RESOURCE_PATH
from stats import sleepStats, combinedStats
from util import utils, plotting as mplot

%load_ext autoreload
%autoreload 2

%matplotlib notebook
sns.set_context("paper")

dataFolder = "path_to_your_fitbit_JSON_export"
statsFolder = join(dataFolder, os.path.pardir, 'folder_name_for_generated_stats')
import plotly.plotly as py
from plotly.offline import init_notebook_mode, enable_mpl_offline, iplot_mpl
import cufflinks as cf
init_notebook_mode(connected=True)
cf.go_offline(connected=True)
enable_mpl_offline()

from ipywidgets import interact, widgets
#from IPython.display import display, clear_output
# Enable logging from Fitbit Analyer code
import logging
logger = logging.getLogger()
logger.setLevel(logging.ERROR)
logger.handlers[0].stream = sys.stdout<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code># helper method to load provided test data
def load_test_sleep_data():
    filepath =  RESOURCE_PATH + "\\unittest\\test_sleep_basic01.csv"
    data1 = utils.loadIntradayData(filepath)
    filepath =  RESOURCE_PATH + "\\unittest\\test_sleep_basic02.csv"
    data2 = utils.loadIntradayData(filepath)
    return [data1, data2]<jupyter_output><empty_output><jupyter_text>Load sleep data from raw JSON export of Fitbit records.<jupyter_code>start = time.time()
#sleepData = utils.loadSleepData(dataFolder) # use for loading your own data
sleepData = load_test_sleep_data() #use for testing
end = time.time()
print("Data loaded in {:.2f}s".format(end - start))
print("Loaded {} dataframes".format(len(sleepData)))
print("{} total entries".format(np.sum([df.size for df in sleepData])))
print("Sample from first dataframe:")
print(sleepData[0].head())<jupyter_output><empty_output><jupyter_text>Generate and Save Stats For the loaded sleep data generate all currently avaiable stats:* **Basic Stats** (sleep values count, sleep efficiency, hours of sleep, total minutes in bed, N max-values for each stat)* **Timing Stats** (first minute asleep, to bed time, wake up time, sleep interval min/max length)* **Intervals Stats** for each day all the sleep intervals lengths* **Intraday Stats** minute to minute report for each day, for the specified sleep values. Total value count, with normalization and centering on specific time.<jupyter_code>start = time.time()
basicAndTimingStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_BASIC_AND_TIMING)
end = time.time()
print("Computed basicAndTimingStats in {:.2f}s".format(end - start))
start = time.time()
intervalsStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_INTERVALS)
end = time.time()
print("Computed intervalsStats in {:.2f}s".format(end - start))
start = time.time()
intradayStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_INTRADAY)
end = time.time()
print("Computed intradayStats in {:.2f}s".format(end - start))
#print(basicAndTimingStats.head())
#print(intervalsStats.head())
#print(intradayStats.head())<jupyter_output><empty_output><jupyter_text>Save generated stats to file<jupyter_code>today = datetime.date.today().strftime("%Y_%m_%d")
basicAndTimingStatsFilepath = os.path.join(statsFolder, "basicAndTimingStats_{}.csv".format(today))
intervalsStatsFilepath = os.path.join(statsFolder, "intervalStats_{}.csv".format(today))
intradayStatsFilepath = os.path.join(statsFolder, "intradayStats_{}.csv".format(today))

basicAndTimingStats.to_csv(basicAndTimingStatsFilepath, index=False)
intervalsStats.to_csv(intervalsStatsFilepath, index=False)
intradayStats.to_csv(intradayStatsFilepath, index=False)<jupyter_output><empty_output><jupyter_text>Basic Stats<jupyter_code># load basic and timing stats
basicAndTimingStatsFilename = "basicAndTimingStats_{}.csv".format(today)
basicAndTimingStats = pd.read_csv(os.path.join(statsFolder, basicAndTimingStatsFilename), 
                    parse_dates=['date', 'to_bed_time', 'wake_up_time'])
basicAndTimingStats.head()
basicAndTimingStats.info()<jupyter_output><empty_output><jupyter_text>Preliminary Stats<jupyter_code># plot preliminary stats (static plot) 
plot = mplot.plotPreliminaryStats(basicAndTimingStats)<jupyter_output><empty_output><jupyter_text>Weekday Stats<jupyter_code># plot preliminary stats (static plot) 
plot = mplot.plotWeekdayStatsSleep(basicAndTimingStats)
plot
# transform static plot using Plotly
iplot_mpl(plot.fig)<jupyter_output><empty_output><jupyter_text>Monthly Stats<jupyter_code>plot = mplot.plotMonthlyStatsSleep(basicAndTimingStats)
plot<jupyter_output><empty_output><jupyter_text>Year-Month Stats<jupyter_code># plot Year-Month stats (static plot)
plot = mplot.plotYearAndMonthStatsSleep(basicAndTimingStats)
# interactive single year-month stat
@interact(stat_name=mplot.NAMES.keys())
def iplot_yearAndMonthStats(stat_name):
    data = basicAndTimingStats.groupby(basicAndTimingStats['date'].dt.to_period("M"))[[stat_name]].mean()
    data.iplot(title='Year-Month Average - {}'.format(mplot.NAMES[stat_name]))<jupyter_output><empty_output><jupyter_text>Weekday Stats By Month<jupyter_code># plot weekday stats by month (static plot) 
plot = mplot.plotWeekdayStatsByMonthSleep(basicAndTimingStats)
weekdayByMonthStats = mplot._prepareWeekdayByMonthStats(basicAndTimingStats)
@interact(stat_name=mplot.NAMES.keys())
def iplot_weekdayByMonthStats(stat_name):
    weekdayByMonthStats.pivot('day', 'month', values=stat_name).iplot(title='Weekday By Month - {}'.format(mplot.NAMES[stat_name]),
                                                    xTitle='Weekday', yTitle=mplot.NAMES[stat_name])<jupyter_output><empty_output><jupyter_text>Daily Stats<jupyter_code>@interact(stat_name=mplot.NAMES.keys())
def iplot_weekdayByMonthStats(stat_name):
    data = basicAndTimingStats[['date', stat_name]].set_index(['date'])
    data.iplot(title='Daily Stats - {}'.format(mplot.NAMES[stat_name]), yTitle=mplot.NAMES[stat_name])<jupyter_output><empty_output><jupyter_text>Intraday Stats<jupyter_code>intradayStatsFilename = "intradayStats_{}.csv".format(today)
intradayStats = pd.read_csv(os.path.join(statsFolder, intradayStats), 
                    parse_dates=['date', 'to_bed_time', 'wake_up_time'])
intradayStats.drop("date", axis=1, inplace=True)
data = intradayStats.apply(pd.value_counts)
#mplot.plotSleepValueHeatmap(data, sleepValue=1)
normIntradayCountStats = sleepStats.normalizedIntradayCountStats(intradayStats)
centeredIntradayCountStats = sleepStats.centerIntradayCountStats(normIntradayCountStats)
#mplot.plotSleepValueHeatmap(centeredIntradayCountStats, sleepValue=1)<jupyter_output><empty_output><jupyter_text>Daily Stats<jupyter_code>#stats.set_index('date', inplace=True)
stats['sleep_efficiency_rol_mean'] = stats['sleep_efficiency'].rolling(center=False,window=20).mean()
stats['sleep_efficiency'].plot()
stats['sleep_efficiency_rol_mean'].plot()
sns.plt.show()
testData = stats['restless']
testData.resample('20D').mean().plot()
testData.plot()
sns.plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11888745148771022, 'fraction_numerical': 0.0028460543337645535, 'mean_word_length': 7.968677494199536, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '251197', 'license': 'Apache-2.0', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2521, 'n_words': 475, 'path': 'Sleep Dashboard.ipynb', 'repo_name': '5agado/fitbit-analyzer', 'size': '7730'}