<commit_before>#!/bin/sh
#
# Bootstrap script which
#
# - Creates a PostGIS DB
# - Initializes osmosis
# - Fills initial OSM data for Switzerland
# - Creates necessary indices
# - Performs a vacuum (really :-))
# - and creates auxiliary OSM views needed by consumer applications

DB_NAME=osmaxx_db
DIR=$(pwd)
WORKDIR_OSM=~/osmaxx/.osmosis
# SQL


execute_sql() {
    psql --dbname $DB_NAME -c "$1" -U postgres
}

setup_db() {
    echo "*** setup DB with postgis extensions ***"
    dropdb -U postgres --interactive --if-exists $DB_NAME
    createdb   -U postgres $DB_NAME
    execute_sql "CREATE EXTENSION hstore;"
    psql -U postgres -d $DB_NAME -f /usr/share/postgresql/9.3/contrib/postgis-2.1/postgis.sql
    psql -U postgres -d $DB_NAME -f /usr/share/postgresql/9.3/contrib/postgis-2.1/spatial_ref_sys.sql
}

perform_vacuum() {
    echo "***** performing vacuum *****" >&2
    execute_sql "VACUUM FREEZE ANALYZE;"
	}

init_osmosis() {
    echo "*** init osmosis ***"
    mkdir -p $WORKDIR_OSM
    osmosis --read-replication-interval-init workingDirectory=$WORKDIR_OSM
    cp $DIR/src/bootstrap-configuration.txt $WORKDIR_OSM/configuration.txt
}

fill_initial_osm_data(){
echo "*** fill initial OSM data ***"
    wget -q http://download.geofabrik.de/europe/switzerland-latest.osm.pbf -O switzerland-latest.osm.pbf
    osm2pgsql --slim --create --extra-attributes --cache-strategy sparse --database $DB_NAME \
        --prefix osm --style $DIR/src/terminal.style --tag-transform-script $DIR/src/style.lua\
        --number-processes 8 --username postgres --hstore-all --input-reader pbf switzerland-latest.osm.pbf
}

cleandata(){
echo 'creating functions...'
psql -U postgres -f ./src/create_functions.sql $DB_NAME
echo 'cleaning database...'
psql -U postgres -f ./src/sweeping_data.sql $DB_NAME
}

filterdata(){
echo 'filtering data...'
     sh filter_data.sh $DB_NAME
}

echo -n 'PostgreSQL admin password: '
read PGPASSWORD
export PGPASSWORD # makes dropdb, createdb & psql not prompt for a password and use this one instead

STARTTIME=$(date +%s)
setup_db && init_osmosis  && fill_initial_osm_data  && cleandata && filterdata
ENDTIME=$(date +%s)
echo "It took $(($ENDTIME - $STARTTIME)) seconds to complete..."<commit_msg>prompt for password silently (without echoing the input)<commit_after>#!/bin/sh
#
# Bootstrap script which
#
# - Creates a PostGIS DB
# - Initializes osmosis
# - Fills initial OSM data for Switzerland
# - Creates necessary indices
# - Performs a vacuum (really :-))
# - and creates auxiliary OSM views needed by consumer applications

DB_NAME=osmaxx_db
DIR=$(pwd)
WORKDIR_OSM=~/osmaxx/.osmosis
# SQL

read_secret()
{
    stty_orig=$(stty -g)
    trap 'stty $stty_orig' EXIT
    stty -echo
    read "$@"
    stty $stty_orig
    trap - EXIT
    echo
}

execute_sql() {
    psql --dbname $DB_NAME -c "$1" -U postgres
}

setup_db() {
    echo "*** setup DB with postgis extensions ***"
    dropdb -U postgres --interactive --if-exists $DB_NAME
    createdb   -U postgres $DB_NAME
    execute_sql "CREATE EXTENSION hstore;"
    psql -U postgres -d $DB_NAME -f /usr/share/postgresql/9.3/contrib/postgis-2.1/postgis.sql
    psql -U postgres -d $DB_NAME -f /usr/share/postgresql/9.3/contrib/postgis-2.1/spatial_ref_sys.sql
}

perform_vacuum() {
    echo "***** performing vacuum *****" >&2
    execute_sql "VACUUM FREEZE ANALYZE;"
	}

init_osmosis() {
    echo "*** init osmosis ***"
    mkdir -p $WORKDIR_OSM
    osmosis --read-replication-interval-init workingDirectory=$WORKDIR_OSM
    cp $DIR/src/bootstrap-configuration.txt $WORKDIR_OSM/configuration.txt
}

fill_initial_osm_data(){
echo "*** fill initial OSM data ***"
    wget -q http://download.geofabrik.de/europe/switzerland-latest.osm.pbf -O switzerland-latest.osm.pbf
    osm2pgsql --slim --create --extra-attributes --cache-strategy sparse --database $DB_NAME \
        --prefix osm --style $DIR/src/terminal.style --tag-transform-script $DIR/src/style.lua\
        --number-processes 8 --username postgres --hstore-all --input-reader pbf switzerland-latest.osm.pbf
}

cleandata(){
echo 'creating functions...'
psql -U postgres -f ./src/create_functions.sql $DB_NAME
echo 'cleaning database...'
psql -U postgres -f ./src/sweeping_data.sql $DB_NAME
}

filterdata(){
echo 'filtering data...'
     sh filter_data.sh $DB_NAME
}

echo -n 'PostgreSQL admin password: '
read_secret PGPASSWORD
export PGPASSWORD # makes dropdb, createdb & psql not prompt for a password and use this one instead

STARTTIME=$(date +%s)
setup_db && init_osmosis  && fill_initial_osm_data  && cleandata && filterdata
ENDTIME=$(date +%s)
echo "It took $(($ENDTIME - $STARTTIME)) seconds to complete..."
{'fraction_non_alphanumeric': 0.13549495383293966, 'fraction_numerical': 0.005153532316942238, 'mean_word_length': 3.5135658914728682, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4579781', 'n_tokens_mistral': 1677, 'n_tokens_neox': 1640, 'n_words': 421}