<jupyter_start><jupyter_text>Get averages from df<jupyter_code>def get_averages(df):
    df['avg_group_size'] = df['sum(num_people)'] / df['sum(num_groups)']
    df['avg_velocity'] = df['sum(sum_velocities)'] / df['sum(num_velocities)']
    df.rename(columns={'avg(num_people)': 'avg_num_people', 'window': 'start',
                       'sum(num_people)': 'sum_num_people', 
                       'sum(num_groups)': 'sum_num_groups',
                       'sum(sum_velocities)': 'sum_window_velocities',
                       'sum(num_velocities)': 'sum_num_velocities'}, 
              inplace=True)
    df.fillna(0., inplace=True)
#     df.drop(columns=['Unnamed: 0', 'sum(num_people)', 'sum(num_groups)',
#                      'sum(sum_velocities)', 'sum(num_velocities)'],
#             inplace=True)
    return df
df = get_averages(df)
df.columns
df[:2]<jupyter_output><empty_output><jupyter_text>Remove<jupyter_code>def filter_window_row(row):
    start, _ = row[4:-1].replace('start=', '').replace('end=', '').split(', datetime.')
    start = start.replace("datetime.", "")
    return eval(start)
df['start'] = df.start.map(filter_window_row)
df.index.name = 'index'
df[:2]<jupyter_output><empty_output><jupyter_text>Aggregate into 24 hour period (i.e. drop dates and convert to time only)<jupyter_code>df['trimmed'] = pd.Series([val.time() for val in df['start']])
df[:2]
grouped = df.groupby([df.start.dt.hour])
summed_df = grouped.sum().drop(columns=['avg_group_size', 'avg_velocity', 'avg_num_people'])
summed_df
# Window size in minutes
window_size = 10
windows_per_hour = 60 / window_size
fps = 2
seconds_in_hour = 3600
frames_per_hour = seconds_in_hour * fps
summed_df['avg_group_size'] = summed_df['sum_num_people'] / summed_df['sum_num_groups']
summed_df['avg_velocity'] = summed_df['sum_window_velocities'] / summed_df['sum_num_velocities']
# Get average number of people per frame
# First, get average within 10-minute window 
# Convert to hour by multiplying by 6
# Then for each hour, there are 3600 seconds, and 3600*2 = frames
summed_df['avg_num_people'] = grouped.sum_num_people.mean() * windows_per_hour / frames_per_hour
summed_df
summed_df.drop(columns=['Unnamed: 0', 'sum_num_people', 'sum_num_groups', 
                        'sum_window_velocities', 'sum_num_velocities'],
               inplace=True)
# Store data as 24x3 matrix
data = np.array([summed_df['avg_group_size'], summed_df['avg_velocity'], summed_df['avg_num_people']]).T
np.savez('frame_averages.npz', data=data)
loaded_data = np.load('frame_averages.npz')['data']
loaded_data<jupyter_output><empty_output><jupyter_text>Display per-frame averages for comparison during streaming for each hour (start)<jupyter_code>summed_df
times = range(24)
fig, axes = plt.subplots(3, 1, figsize=(8, 14))

axes[0].plot(times, data[:, 0], 'b.-')
axes[0].set_title('Average group size per frame for each hour')
axes[0].set_xticks(range(24))

axes[1].plot(times, data[:, 1], 'b.-')
axes[1].set_title('Average velocity per frame for each hour')
axes[1].set_xticks(range(24))

axes[2].plot(times, data[:, 2], 'b.-')
axes[2].set_title('Average number of people per frame for each hour')
axes[2].set_xlabel('Hour of the day')
axes[2].set_xticks(range(24))
fig.savefig('perframeanalytics.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1590909090909091, 'fraction_numerical': 0.01818181818181818, 'mean_word_length': 5.056880733944954, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '618154', 'license': 'MIT', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1179, 'n_words': 220, 'path': 'notebooks/HistoricalComparison.ipynb', 'repo_name': 'stephenslater/CS205-Parallelism', 'size': '3300'}