<commit_before>#!/bin/bash

set -eu

TMP=${TMP:-'/tmp/gnaf'}
PGUSER='postgres'
DBNAME='gnaf'
PSQL="psql -t -q -U $PGUSER $DBNAME"

# set up postgres, directories
mkdir -p $TMP/gnaf
mkdir -p $TMP/gnaf-admin
dropdb -U $PGUSER --if-exists $DBNAME
createdb -U $PGUSER $DBNAME
echo "CREATE EXTENSION postgis;" | $PSQL

# fetch data/resources
git clone <EMAIL>:minus34/gnaf-loader.git $TMP/gnaf-loader &
curl -s 'https://s3-ap-southeast-2.amazonaws.com/datagovau/FEB16_AdminBounds_ESRI.zip' -o $TMP/gnaf-admin.zip &
curl -s 'https://s3-ap-southeast-2.amazonaws.com/datagovau/FEB16_GNAF%2BEULA_PipeSeparatedValue_20160222170142.zip' -o $TMP/gnaf.zip &
wait
parallel "unzip -d $TMP/{} $TMP/{}.zip" ::: gnaf gnaf-admin

# find file directories
GNAF_DIR="$(find $TMP -type d | grep 'G-NAF' | grep 'Authority Code' | xargs -I {} dirname {} | head -n1)"
BOUNDARY_DIR="$(find $TMP -type d | grep 'AdminBounds_ESRI' | grep -v 'Administrative Boundaries' | head -n1)"

# load data into tables
python $TMP/gnaf-loader/load-gnaf.py \
    --pguser $PGUSER \
    --pgdb $DBNAME \
    --gnaf-tables-path $GNAF_DIR
    --admin-bdys-path $BOUNDARY_DIR

# select output from tables
echo "CREATE TABLE openaddresses AS
SELECT

    TRIM(
        CASE WHEN number_first IS NOT NULL THEN number_first ||
            CASE WHEN number_last IS NOT NULL THEN '-' || number_last || ' ' ELSE ' ' END
        ELSE
            NULL<commit_msg>update gnaf script for MAY16 release<commit_after>#!/bin/bash

# requires: postgis unzip parallel curl git python-psycopg2 zip

set -eu

TMP=${TMP:-'/tmp/gnaf'}
PGUSER='postgres'
DBNAME='gnaf'
PSQL="psql -t -q -U $PGUSER $DBNAME"

# set up postgres, directories
mkdir -p $TMP/gnaf
mkdir -p $TMP/gnaf-admin
dropdb -U $PGUSER --if-exists $DBNAME
createdb -U $PGUSER $DBNAME
echo "CREATE EXTENSION postgis;" | $PSQL

# fetch data/resources
git clone <EMAIL>:minus34/gnaf-loader.git $TMP/gnaf-loader &
curl -s 'https://s3-ap-southeast-2.amazonaws.com/datagovau/MAY16_AdminBounds_ESRIShapefileorDBFfile_20160523140152.zip' -o $TMP/gnaf-admin.zip &
curl -s 'https://s3-ap-southeast-2.amazonaws.com/datagovau/MAY16_GNAF%2BEULA_PipeSeparatedValue_20160523140820.zip' -o $TMP/gnaf.zip &
wait
parallel "unzip -d $TMP/{} $TMP/{}.zip" ::: gnaf gnaf-admin

# find file directories
GNAF_DIR="$(find $TMP -type d | grep 'G-NAF' | grep 'Authority Code' | xargs -I {} dirname {} | head -n1)"
BOUNDARY_DIR="$(find $TMP -type d | grep -v 'Administrative Boundaries' | head -n1)"

# load data into tables
python $TMP/gnaf-loader/load-gnaf.py \
    --pguser $PGUSER \
    --pgdb $DBNAME \
    --gnaf-tables-path "$GNAF_DIR" \
    --admin-bdys-path "$BOUNDARY_DIR" \
    --raw-unlogged

# select output from tables
echo "CREATE TABLE openaddresses AS
SELECT

    TRIM(
        CASE WHEN number_first IS NOT NULL THEN number_first ||
            CASE WHEN number_last IS NOT NULL THEN '-' || number_last || ' ' ELSE ' ' END
        ELSE
            NULL
{'fraction_non_alphanumeric': 0.14329475833900612, 'fraction_numerical': 0.024166099387338325, 'mean_word_length': 3.4395770392749245, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6770527', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1168, 'n_words': 279}