<jupyter_start><jupyter_text>This notebook is for querying the log of the station status messages sent to the server This file is stored on the server at: /home/lma_admin/status_logs.csv<jupyter_code>%pylab inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import csv
log_file = '/Users/Vanna/AppData/Local/Temp/sublime-sftp-browse-1456937310/lma_admin/home/lma_admin/status_logs.csv'<jupyter_output><empty_output><jupyter_text>The primary interests in this file are listed in the columns and ingested into the dataframe, but others exist in the same order that they exist on the status webpage<jupyter_code>columns = ['Time','Station','CPU Load 1', 'CPU Load 2','CPU Load 3',
           'Boot','Root','shm','Data','GPS','Threshold','Triggers',
           'Temp','PV Voltage','Battery Voltage','Load']
test = pd.read_csv(log_file, 
                   usecols=[1,3,4,5,6,7,9,10,11,12,22,28,29,31,36,37,38], 
                   header=None,parse_dates=[[3,4]],
#                   skiprows=1 # For use if the initial line is missing categories
                  )
test.columns=columns
test = test.set_index(['Station','Time'])<jupyter_output><empty_output><jupyter_text>If case any specific column was not read in as a float:<jupyter_code># test['Battery Voltage'] = test['Battery Voltage'].astype('float64')
# test['Load'] = test['Load'].astype('float64')
# test['PV Voltage'] = test['PV Voltage'].astype('float64')<jupyter_output><empty_output><jupyter_text>A few different plotting methods<jupyter_code>thing_to_plot    = 'Threshold'
interval_to_plot = datetime.datetime.now() - datetime.timedelta(days=2)<jupyter_output><empty_output><jupyter_text>Specifically specified:<jupyter_code>plt.plot(test[thing_to_plot]['Biggin'][interval_to_plot:],color='darkblue',label='Biggin Hill')
plt.plot(test[thing_to_plot]['Abernathy'][interval_to_plot:],color='c',label='Abernathy')
plt.plot(test[thing_to_plot]['Roosevelt'][interval_to_plot:],color='orange',label='Roosevelt')
plt.plot(test[thing_to_plot]['Llano'][interval_to_plot:],color='k',label='Llano')
plt.plot(test[thing_to_plot]['Petersburg'][interval_to_plot:],color='y',label='Petersburg')
plt.plot(test[thing_to_plot]['Idalou'][interval_to_plot:],color='darkred',label='Idalou')
plt.plot(test[thing_to_plot]['Reese'][interval_to_plot:],color='b',label='Reese')
plt.plot(test[thing_to_plot]['Lorenzo'][interval_to_plot:],color='green',label='Lorenzo')
plt.plot(test[thing_to_plot]['NewDeal'][interval_to_plot:],color='lightblue',label='New Deal')
# plt.plot(test[thing_to_plot]['Wolfforth'][interval_to_plot:],color='r',label='Wolfforth')
# plt.plot(test[thing_to_plot]['Estacado'][interval_to_plot:])
# plt.plot(test[thing_to_plot]['Shallowater'][interval_to_plot:])

plt.legend()
plt.xticks(rotation=45)
plt.show()<jupyter_output><empty_output><jupyter_text>Less specified:<jupyter_code>color=iter(cm.spectral(np.linspace(0,1,13)))
for i in range(len(test.index.levels[0])):
    c = next(color)
    test.loc[test.index.levels[0][i]][thing_to_plot].plot(label=test.index.levels[0][i],c=c)

plt.xlim((interval_to_plot,datetime.datetime.now()))
plt.legend()
plt.xticks(rotation=45)
plt.show()<jupyter_output><empty_output><jupyter_text>Individual queries:<jupyter_code>test.loc['Joppa']['Triggers'].plot()
test.loc['Pulaski']['Triggers'].plot()
# pd.rolling_mean(test.groupby(level='Time').mean()[thing_to_plot], 30).plot()
# pd.rolling_mean(test.groupby(level='Time').std()[thing_to_plot], 3).plot()

plt.xlim((interval_to_plot,datetime.datetime.now()+datetime.timedelta(hours=6)))
plt.show()
test.loc['Joppa']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17486187845303866, 'fraction_numerical': 0.017955801104972375, 'mean_word_length': 8.579365079365079, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '602945', 'license': 'MIT', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1321, 'n_words': 167, 'path': 'Status_Messages_Query.ipynb', 'repo_name': 'vbalderdash/LMA_QC', 'size': '3620'}