<commit_before>#!/bin/bash
# This is a script run every half hour to scrape current observations
# It is run through crontab, editable with:
# sudo crontab -u shiny -e

# download latest observations 
curl -o /srv/isithotrightnow/data/IDN60901.94768.axf http://www.bom.gov.au/fwo/IDN60901/IDN60901.94768.axf
curl -o /srv/isithotrightnow/data/IDN60901.94768.json http://www.bom.gov.au/fwo/IDN60901/IDN60901.94768.json

# grep last 30min obs (line starting with '0,') into data/hist_ file
grep -o '^0,.*' /srv/isithotrightnow/data/IDN60901.94768.axf >> /srv/isithotrightnow/data/hist_IDN60901.94768.csv

echo "latest observations have been scraped"
exit<commit_msg>updated cronscripts for Melbourne and Brisbane<commit_after>#!/bin/bash
# This is a script run every half hour to scrape current observations
# It is run through crontab, editable with:
# sudo crontab -u shiny -e

# download latest observations

### NEW SOUTH WALES (N) ###
# Sydney Obsevatory
curl -o /srv/isithotrightnow/data/IDN60901.94768.axf http://www.bom.gov.au/fwo/IDN60901/IDN60901.94768.axf
curl -o /srv/isithotrightnow/data/IDN60901.94768.json http://www.bom.gov.au/fwo/IDN60901/IDN60901.94768.json

### VICTORIA (V) ###
# Melbourne (Olympic Park)
curl -o /srv/isithotrightnow/data/IDV60901.95936.axf http://www.bom.gov.au/fwo/IDV60901/IDV60901.95936.axf
curl -o /srv/isithotrightnow/data/IDV60901.95936.json http://www.bom.gov.au/fwo/IDV60901/IDV60901.95936.json

### QUEENSLAND (Q) ###
# Brisbane
curl -o /srv/isithotrightnow/data/IDQ60901.94576.axf http://www.bom.gov.au/fwo/IDQ60901/IDQ60901.94576.axf
curl -o /srv/isithotrightnow/data/IDQ60901.94576.json http://www.bom.gov.au/fwo/IDQ60901/IDQ60901.94576.json

# grep last 30min obs (line starting with '0,') into data/hist_ file
grep -o '^0,.*' /srv/isithotrightnow/data/IDN60901.94768.axf >> /srv/isithotrightnow/data/hist_IDN60901.94768.csv
grep -o '^0,.*' /srv/isithotrightnow/data/IDV60901.95936.axf >> /srv/isithotrightnow/data/hist_IDV60901.95936.csv
grep -o '^0,.*' /srv/isithotrightnow/data/IDQ60901.94576.axf >> /srv/isithotrightnow/data/hist_IDQ60901.94576.csv

echo "latest observations have been scraped"
exit
{'fraction_non_alphanumeric': 0.1417945141794514, 'fraction_numerical': 0.13482101348210135, 'mean_word_length': 4.420654911838791, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2942801', 'n_tokens_mistral': 1106, 'n_tokens_neox': 927, 'n_words': 162}