<jupyter_start><jupyter_text>Notes<jupyter_code>notes_df = pd.read_csv('data/notes.csv')
notes_df
df = notes_df
df["Day"] = pd.to_datetime(df['Timestamp']).dt.date
df
# # https://pandas.pydata.org/pandas-docs/stable/getting_started/intro_tutorials/09_timeseries.html
# grouped_notes = average_notes.groupby([pd.to_datetime(df["Day"]).dt.date])
# grouped_notes = pd.DataFrame(grouped_notes)
# grouped_notes

# sum_of_notes_per_day = []
# for note in grouped_notes[1]:
#     sum_of_notes_per_day.append(len(note))
# sum_of_notes_per_day
by_day = pd.to_datetime(notes_df['Day']).dt.to_period('D').value_counts().sort_index()
by_day.index = pd.PeriodIndex(by_day.index)
df_day = by_day.rename_axis('Day').reset_index(name='Counts')
df_day
df['Counts'] = 0
df.head(5)
df_day['Day'] = df_day['Day'].astype(str)
df_day['Day'] = pd.to_datetime(df_day['Day']).dt.date
print(type(df_day['Day'][0]))
print(type(df['Day'][0]))
print()
print(df.dtypes)
print()
print(df_day.dtypes)
df_day['Counts'].max()
df_day['Counts'].mean()
df_day['Counts'].median()
df_day['Counts'].std()
# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.merge.html
df = pd.merge(df, df_day, how='right', on='Day')
df
# a1, a2 = df.align(df_day, join='left', axis=1)
# a1
# a2
# https://stackoverflow.com/questions/62672427/how-do-i-convert-iso-8601-object-into-datetime-format-in-a-pandas-dataframe
df['Timestamp'] = pd.to_datetime(df['Timestamp'])
df.head(5)
df["Timestamp"] = df["Timestamp"].apply(lambda x: x.replace(year=1970, month=1, day=1))
df['Timestamp'] = pd.to_datetime(df['Timestamp'])
df
# https://stackoverflow.com/questions/52007139/get-the-average-date-from-multiple-dates-pandas
average_notes = pd.DataFrame()
average_notes['Timestamp'] = [x.value for x in df['Timestamp']]
print(average_notes.head(5))
average_notes.describe()
# https://stackoverflow.com/questions/19231871/convert-unix-time-to-readable-date-in-pandas-dataframe
# https://stackoverflow.com/questions/54733465/pandas-how-to-extract-hhmm-from-datetime-column-in-python/54733547
# https://stackoverflow.com/questions/39662149/pandas-extract-date-and-time-from-timestamp

mean_notes = pd.to_datetime(average_notes['Timestamp'].mean())
mean_notes_time = mean_notes.strftime('%H:%M:%S')
print(f"Mean Notes: {mean_notes}")

median_notes = pd.to_datetime(average_notes['Timestamp'].median())
median_notes_time = median_notes.strftime('%H:%M:%S')
print(f"Median Notes: {median_notes}")
rolling=7
rolling_timestamp = average_notes.rolling(rolling).mean()
rolling_timestamp['Timestamp'] = rolling_timestamp.apply(pd.to_datetime)
rolling_timestamp.head(10)
print(type(rolling_timestamp['Timestamp'][38]))
rolling_timestamp['Timestamp'][38]
# for time in rolling_timestamp['Timestamp'][6:]:
#     time = time.strftime('%H:%M:%S')
# print(rolling_timestamp['Timestamp'].head(10))
df['Day'][0]
df['Day'] = df['Day'].apply(str)
df['Day']
df<jupyter_output><empty_output><jupyter_text>Creating '\' in notes<jupyter_code>counts = []
notes = []
for note in df['Have an idea? Need to remember something?']:
    counts.append(len(note.split()))
    notes.append(note.split())
notes[:3]
print(counts)
df['Word Count'] = counts
df
# https://stackoverflow.com/questions/31040525/insert-element-in-python-list-after-every-nth-element
note2 = []
for note in notes:
    note = [x for y in (note[i:i+7] + ['<br>'] * (i < len(note) - 1) for i in range(0, len(note), 7)) for x in y]
    note2.append(note)
note2[:3]
s = ' '
note3 = []
for note in note2:
    note3.append(s.join(note))
    
note3[:3]
df['Have an idea? Need to remember something?'] = note3
df['Have an idea? Need to remember something?']
df['Rolling Timestamp'] = rolling_timestamp
df
# can add other customdata to the stack, maybe number of words per note?
customdata = np.stack((df['Counts_y'], 
                       df['Have an idea? Need to remember something?'],
                       df['Word Count']),
                       axis=1)
print(customdata.shape)
customdata[-3:]<jupyter_output>(143, 3)<jupyter_text>Dashboard<jupyter_code># https://community.plotly.com/t/how-do-i-change-the-x-axis-representation-of-time-on-the-px-timeline/44834/4
# https://stackoverflow.com/questions/67319505/how-to-plot-multiple-timelines-in-one-graph
# https://stackoverflow.com/questions/66905297/plotly-is-it-possible-to-add-a-second-y-axis-to-plotly-timeline

title = 'Notes Dashboard'

# rolling=7
# rolling_timestamp = df['Timestamp'].rolling(rolling).mean()

fig = go.Figure()
fig = px.timeline(df, x_start="Timestamp", x_end="Timestamp", y='Day', )

# https://plotly.com/python/marker-style/
fig.add_trace(go.Scatter(x=df['Timestamp'], 
                         y=df['Day'], 
                         name='Timestamp', 
                         mode='markers',
                         marker_color='#636EFA',
                         
                         # https://chart-studio.plotly.com/~empet/15366/customdata-for-a-few-plotly-chart-types/#/
                         # https://plotly.com/python/hover-text-and-formatting/
                         customdata = customdata, 
                         hovertemplate =
                         "Timestamp: %{x} %{y}<br>" +
                         "Sum of Notes for Day: %{customdata[0]}<br>" +
                         "Word Count of Note: %{customdata[2]}<br>" +
                         "%{customdata[1]}"
                         "<extra></extra>",
                        ))

fig.add_trace(go.Scatter(x=df['Rolling Timestamp'], 
                         y=df['Day'], 
                         name=f'{rolling} Day Rolling Timestamp',
                         customdata = customdata, 
                         hovertemplate =
                         "{rolling} Day Rolling Mean Timestamp: %{x}<br>" +
                         "<extra></extra>",
                        ))

fig.add_vline(x=mean_notes, line_dash="dash")
fig.add_vline(x=median_notes, line_dash="dash")

# https://github.com/d3/d3-time-format#locale_format change time format
fig.update_layout(xaxis=dict(
                      title='Timestamp', 
                      tickformat = '%H:%M:%S',
                      range=['1970-01-01 00:00:00','1970-01-01 23:59:59']
                  ),
                  yaxis=dict(
                      title='Day',
                      tickformat = '%B/%d/%Y',
                      dtick="M1",
                      #dtick=86400000*2
                   ))

fig.update_layout(
    #hovermode='x unified',
    #dragmode='pan',
    annotations=[
        #https://plotly.com/python/text-and-annotations/#adding-annotations-with-xref-and-yref-as-paper
        #https://community.plotly.com/t/how-to-add-a-text-area-with-custom-text-as-a-kind-of-legend-in-a-plotly-plot/24349/3
        go.layout.Annotation(
            text=f'Mean Notes: {mean_notes_time}<br>Median Notes: {median_notes_time}',
            align='left',
            showarrow=False,
            xref='paper',
            yref='paper',
            yanchor="top",
            y=.98,
            xanchor="right",
            x=1,)])
    
fig.update_layout(#hovermode='x unified',
                  #dragmode='pan',
                  title=dict(text=title, 
                             yanchor="top",
                             y=.98,
                             xanchor="left",
                             x=.05),
                 legend=dict(orientation="h",
                             yanchor="bottom",
                             y=1.02,
                             xanchor="right",
                             x=1))
fig.update_yaxes(showspikes=True)
fig.update_yaxes(autorange="reversed", # otherwise tasks are listed from the bottom up
                 showspikes=True)

app = jupyter_dash.JupyterDash(__name__,
                               external_stylesheets=external_stylesheets,
                               title=f"{title}")

app.layout = html.Div([
    dcc.Graph(
        figure=fig,
        
        #https://plotly.com/python/setting-graph-size/
        #https://stackoverflow.com/questions/46287189/how-can-i-change-the-size-of-my-dash-graph
        style={'height': '98vh',
               #'width': '150vh',
              }
    ),
])

if __name__ == '__main__':
    app.run_server(mode='external', port=8060, debug=True)<jupyter_output>Dash app running on http://127.0.0.1:8060/<jupyter_text>Histogram<jupyter_code>title = "Notes Histogram"

fig = go.Figure()
fig.add_trace(go.Histogram(x=df['Counts_y'], 
                           name="Sum of Notes for Day",
                           hovertemplate =
                           "Sum of Notes for Day: %{x}<br>" +
                           "Number of Notes with that sum: %{y}<br>"
                           "<extra></extra>",))
fig.add_trace(go.Histogram(x=df['Word Count'], 
                           name="Number of Words in Note",
                           hovertemplate =
                           "Number of Words in Note: %{x}<br>" +
                           "Number of Notes with that Word Count: %{y}<br>"
                           "<extra></extra>",
                           marker_color='#00CC96',))

# Overlay both histograms
fig.update_layout(barmode='overlay')

# Reduce opacity to see both histograms
fig.update_traces(opacity=0.75, 
                  # https://plotly.com/python/reference/histogram/#histogram-xbins
                  xbins_size=1,
                  #xbins_start=60000,
                  #xbins_end=86394000,
                 )

app = jupyter_dash.JupyterDash(__name__,
                               external_stylesheets=external_stylesheets,
                               title=f"{title}")

app.layout = html.Div([
    dcc.Graph(
        figure=fig,
        
        #https://plotly.com/python/setting-graph-size/
        #https://stackoverflow.com/questions/46287189/how-can-i-change-the-size-of-my-dash-graph
        style={'height': '98vh',
               #'width': '150vh',
              }
    ),
])

if __name__ == '__main__':
    app.run_server(mode='external', port=9000, debug=True)<jupyter_output>Dash app running on http://127.0.0.1:9000/
{'fraction_non_alphanumeric': 0.1554917302168961, 'fraction_numerical': 0.027532930573437656, 'mean_word_length': 2.3348745046235138, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 22, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '446146', 'license': 'MIT', 'n_tokens_mistral': 3499, 'n_tokens_neox': 3227, 'n_words': 462, 'path': 'google_sheets/google_sheets_analysis.ipynb', 'repo_name': 'mcwaage1/qs', 'size': '10097'}