<commit_before>#!/bin/bash
set -e

timestamp=`date +%Y%m%d%H%M`
filename=transit-${timestamp}.db

[ -f google_transit.zip ] && rm  google_transit.zip
[ -d feed ] && rm -r feed
curl -O http://www.transperth.wa.gov.au/TimetablePDFs/GoogleTransit/google_transit.zip
mkdir feed
cd feed
unzip ../google_transit.zip
cd ..
cat gtfs_tables.sqlite <(python import_gtfs_to_sql.py feed nocopy) | sqlite3 $filename
cat post-import.sql | sqlite3 $filename
echo 'CREATE TABLE shouldirun_config (name, value);' | sqlite3 $filename
echo "INSERT INTO shouldirun_config VALUES ('version', '$timestamp');" | sqlite3 $filename
echo 'VACUUM;' | sqlite3 $filename
sha512=(`shasum -a 512 $filename`)
gzip $filename
cat << EOF > $timestamp.json
{
    "version": "$timestamp",
    "url": "",
    "sha512": "$sha512",
    "description": "Timetable update from Transperth. Updated `date`."
}
EOF<commit_msg>Make the import script clever<commit_after>#!/bin/bash
set -e

timestamp=`date +%Y%m%d%H%M`
timestampfilename=`date +%Y-%m-%d-%H%M`
filename=transit-${timestamp}.db

[ -f google_transit.zip ] && rm  google_transit.zip
[ -d feed ] && rm -r feed
curl -O http://www.transperth.wa.gov.au/TimetablePDFs/GoogleTransit/google_transit.zip
mkdir feed
cd feed
unzip ../google_transit.zip
cd ..
cat gtfs_tables.sqlite <(python import_gtfs_to_sql.py feed nocopy) | sqlite3 $filename
cat post-import.sql | sqlite3 $filename
echo 'CREATE TABLE shouldirun_config (name, value);' | sqlite3 $filename
echo "INSERT INTO shouldirun_config VALUES ('version', '$timestamp');" | sqlite3 $filename
echo 'VACUUM;' | sqlite3 $filename
sha512=(`shasum -a 512 $filename`)
gzip $filename
cat << EOF > $timestampfilename.json
{
    "version": "$timestamp",
    "url": "",
    "sha512": "$sha512",
    "description": "Timetable update from Transperth. Updated `date`."
}
EOF
echo "Now Run:"
echo "s3cmd put --acl-public {$filename}.gz s3://shouldirun"
echo "s3cmd info s3://shouldirun/{$filename}.gz"
{'fraction_non_alphanumeric': 0.15743589743589745, 'fraction_numerical': 0.019487179487179488, 'mean_word_length': 4.858858858858859, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2317738', 'n_tokens_mistral': 737, 'n_tokens_neox': 685, 'n_words': 178}