<commit_before>MAILTO=REPLACE
0 9 1 * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 9 15 * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

# Do others a little later after the URL list has been downloaded.
0 11 1 * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

0 11 1 * * cd /var/www/httparchive.android/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.android/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.android/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

0 11 1 * * cd /var/www/httparchive.chrome/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.chrome/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.chrome/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

# Check that the crawls are running
MAILTO=REPLACE
0 12 1 * * cd /var/www/httparchive.dev/bulktest ; /usr/bin/php batch_check.php
0 12 15 * * cd /var/www/httparchive.dev/bulktest ; /usr/bin/php batch_check.php
25 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_monitor.php
55 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_monitor.php

# Hourly archive pass to archive data to the Internet Archive data store
MAILTO=REPLACE
45 * * * * cd /var/www/webpagetest/cli/ && /usr/bin/php -c /etc/php5/cli/php.ini archive-ia.php > /dev/null 2>&1
0,15,30,45 * * * * cd /var/www/webpagetest/work && /usr/bin/php -c /etc/php5/cli/php.ini checklocations.php > /dev/null 2>&1
<commit_msg>Now chrome and android are the main (dev & mobile) crawls, so we do IE and iphone as ancillary crawls for another month or two. Some new cronjobs for Pat.<commit_after>MAILTO=REPLACE
0 9 1 * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 9 15 * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

# Do others a little later after the URL list has been downloaded.
0 11 1 * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.mobile/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

0 11 1 * * cd /var/www/httparchive.iphone/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.iphone/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.iphone/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

0 11 1 * * cd /var/www/httparchive.ie/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
0 11 15 * * cd /var/www/httparchive.ie/bulktest/ ; /usr/bin/php batch_start.php 2> batch.log
*/30 * * * * cd /var/www/httparchive.ie/bulktest/ ; /usr/bin/php batch_process.php 2>> ./batch.log

# Check that the crawls are running
MAILTO=REPLACE
0 12 1 * * cd /var/www/httparchive.dev/bulktest ; /usr/bin/php batch_check.php
0 12 15 * * cd /var/www/httparchive.dev/bulktest ; /usr/bin/php batch_check.php
25 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_monitor.php
55 * * * * cd /var/www/httparchive.dev/bulktest/ ; /usr/bin/php batch_monitor.php

# Hourly archive pass to archive data to the Internet Archive data store
MAILTO=REPLACE
0,15,30,45 * * * * cd /var/www/wptarchive/ && /usr/bin/php -c /etc/php5/cli/php.ini prune.php > /dev/null 2>&1
0 * * * * cd /var/www/wptarchive/ && /usr/bin/php -c /etc/php5/cli/php.ini scan_tests.php > /dev/null 2>&1
5 * * * * cd /var/www/wptarchive/ && /usr/bin/php -c /etc/php5/cli/php.ini archive.php > /dev/null 2>&1
35 * * * * cd /var/www/wptarchive/ && /usr/bin/php -c /etc/php5/cli/php.ini har.php > /dev/null 2>&1
0 * * * * cd /var/www/wptarchive/ && /usr/bin/php -c /etc/php5/cli/php.ini sql_dumps.php > /dev/null 2>&1
0,15,30,45 * * * * cd /var/www/webpagetest/work && /usr/bin/php -c /etc/php5/cli/php.ini checklocations.php > /dev/null 2>&1

{'fraction_non_alphanumeric': 0.16828836797877045, 'fraction_numerical': 0.04113224237063246, 'mean_word_length': 3.1193078324225865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 40, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6232770', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1793, 'n_words': 414}