<jupyter_start><jupyter_text>This notebook reports how many hours members spent off the board. It provides a table representation of the data as well as a visual heat map representation<jupyter_code>from __future__ import absolute_import, division, print_function
from multiprocessing import Pool

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import sys
sys.path.insert(0, '../../src/data/')

from config import *
#Dates that will be relevant
PERIOD_1 = ['2018-06-12', '2018-06-13', '2018-06-14', '2018-06-15', '2018-06-16', '2018-06-17',
'2018-06-18', '2018-06-19', '2018-06-20', '2018-06-21', '2018-06-22', '2018-06-23', 
'2018-06-24', '2018-06-25', '2018-06-26', '2018-06-27', '2018-06-28', '2018-06-29']
PERIOD_2 = ['2018-07-30', '2018-07-31', '2018-08-01', '2018-08-02', '2018-08-03', '2018-08-04', 
'2018-08-05', '2018-08-06', '2018-08-07', '2018-08-08', '2018-08-09', '2018-08-10', 
'2018-08-11', '2018-08-12', '2018-08-13', '2018-08-14', '2018-08-15']

ALL_DATES = PERIOD_1 + PERIOD_2

WEEKENDS = ['2018-06-16', '2018-06-17', '2018-06-23', '2018-06-24']
#Preparing the members and beacons metadata
members_metadata = pd.read_csv(members_metadata_path).set_index('member')
beacons_metadata = pd.read_csv(beacons_metadata_path).set_index('beacon')<jupyter_output><empty_output><jupyter_text>Small Functions that help filter the data:<jupyter_code>def get_closest_beacon(m5cb, members_metadata, beacons_metadata):
    """
    Parameters: dataframe, dataframe, dataframe
    Returns: a table with the closest beacon for each member
    """
    
    #Create table with only closest beacon
    m1cb = m5cb[['beacon_0', 'rssi_0']].rename(
        columns = {'beacon_0': 'beacon', 'rssi_0': 'rssi'}).reset_index()

    #Add beacon metadata
    m1cb = m1cb.join(
            beacons_metadata[['company', 'type']], on='beacon').rename(
            columns = {'company': 'beacon_company', 'type': 'beacon_type'})
    #Add badge metadata
    m1cb = m1cb.join(
            members_metadata[['company']], on='member').rename(columns = {'company': 'member_company'})
    
    return m1cb
def drop_board_beacons(mySeries):
    """
    Given a dataframe
    Returns a dataframe without board beacons
    """
    return mySeries[mySeries.beacon_type != 'board']
def on_board(mySeries):
    """
    Given a dataframe
    Returns an additional column of booleans
        True if beacon type is board
        False otherwise
    """
    return mySeries['beacon_type'] == 'board'
def count_to_hours(counter):
    """
    Given a count
    Returns hours
    """
    seconds = counter*15
    minutes = seconds/60
    hours = minutes/60
    return(round(hours,2))<jupyter_output><empty_output><jupyter_text>Generating a dataframe of hours off the board for the entirety of period one:<jupyter_code>members_metadata_copy = members_metadata.reset_index()
hours_off_board_period_one = pd.DataFrame(index = members_metadata_copy['member'])

for date in PERIOD_1:
    tempHours = pd.DataFrame()
    
    day_start = date + ' 00:00:00-04:00'
    day_end = date + ' 23:59:30-04:00'
    desired_range = "datetime >= day_start & datetime < day_end"
    
    m5cb = pd.read_hdf(clean_store_path,'proximity/member_5_closest_beacons', where = desired_range, data_columns= True)
    
    m1cb_with_boards = get_closest_beacon(m5cb, members_metadata, beacons_metadata)
    m1cb_with_boards['on_board'] = on_board(m1cb_with_boards)
    
    grouped_with_boards = m1cb_with_boards.groupby(['member', 'on_board'])

    counts_with_boards = grouped_with_boards['on_board'].count()
    counts_with_boards = counts_with_boards.unstack()

    tempHours[date] = counts_with_boards[False].apply(count_to_hours)

    hours_off_board_period_one = hours_off_board_period_one.join(tempHours)<jupyter_output><empty_output><jupyter_text>Generating a dataframe of hours off the board for the entirety of period two<jupyter_code>members_metadata_copy = members_metadata.reset_index()
hours_off_board_period_two = pd.DataFrame(index = members_metadata_copy['member'])

for date in PERIOD_2:
    tempHours = pd.DataFrame()
    
    day_start = date + ' 00:00:00-04:00'
    day_end = date + ' 23:59:30-04:00'
    desired_range = "datetime >= day_start & datetime < day_end"
    
    m5cb = pd.read_hdf(clean_store_path,'proximity/member_5_closest_beacons', where = desired_range, data_columns= True)
    
    m1cb_with_boards = get_closest_beacon(m5cb, members_metadata, beacons_metadata)
    m1cb_with_boards['on_board'] = on_board(m1cb_with_boards)
    
    grouped_with_boards = m1cb_with_boards.groupby(['member', 'on_board'])

    counts_with_boards = grouped_with_boards['on_board'].count()
    counts_with_boards = counts_with_boards.unstack()

    tempHours[date] = counts_with_boards[False].apply(count_to_hours)

    hours_off_board_period_two = hours_off_board_period_two.join(tempHours)
    
#Cleaning the tables
hours_off_board_period_one = hours_off_board_period_one.fillna(0)
hours_off_board_period_two = hours_off_board_period_two.fillna(0)
hours_off_board_period_one.sort_index(inplace = True)
hours_off_board_period_two.sort_index(inplace = True)
#Creating Period One Heatmap
plt.figure(figsize=(20,45))
sns.heatmap(hours_off_board_period_one,cmap='Reds', vmax = 10, 
            cbar = True)
plt.xticks(rotation = 45)
plt.show()
#Creating Period Two Heatmap
plt.figure(figsize=(20,45))
sns.heatmap(hours_off_board_period_two,cmap='Reds', vmax = 10,
            cbar = True)
plt.xticks(rotation = 50)
plt.show()
#Appending to the tables
hours_off_board_period_one['Total_Off_Board'] = hours_off_board_period_one.sum(axis=1)
hours_off_board_period_two['Total_Off_Board'] = hours_off_board_period_two.sum(axis=1)
hours_off_board_period_one.head()
hours_off_board_period_two.head()<jupyter_output><empty_output><jupyter_text>Find and individual members hours off the board for the period<jupyter_code>member = '328SASHRV6'
hours_off_board_period_two.loc[member]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12483288770053476, 'fraction_numerical': 0.06951871657754011, 'mean_word_length': 5.0030090270812435, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '76947', 'license': 'MIT', 'n_tokens_mistral': 2429, 'n_tokens_neox': 2155, 'n_words': 399, 'path': 'notebooks/00-tests/misc/01.10-hours-off-board.ipynb', 'repo_name': 'HumanDynamics/rhythm-public-analysis-deltav18-anon', 'size': '5984'}