<commit_before># Imports all GSD shape data to a database.
# NOTE: If the db exists, it is *dropped* before importing any data

DB=lmv
GSD_PATH=/media/local/MapData/lmv-shape
SQL_LOG=/dev/null
ERROR_LOG=gsd_import.log

# Create a clean slate
rm $SQL_LOG
rm $ERROR_LOG
dropdb $DB

createdb -T template_postgis $DB

# Create tables for Terrängkartan before actually inserting to them
echo Creating structure for Terrängkartan
for F in $GSD_PATH/terrang/south/*.shp; do
    TABLE_NAME=terrang_`basename $F .shp | cut -d_ -f1`
    /usr/lib/postgresql/9.1/bin/shp2pgsql -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Terrängkartan, using text labels from gistext directory
FILES="`find $GSD_PATH/terrang/ -maxdepth 2 -name "*.shp" -not \( -name "tx_*.shp" -or -name "tl_*.shp" \)` `find $GSD_PATH/terrang/ -mindepth 3 \( -name "tx_*.shp" -or -name "tl_*.shp" \)`"
for F in $FILES; do
    TABLE_NAME=terrang_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    /usr/lib/postgresql/9.1/bin/shp2pgsql -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Create tables for Tätortskartan before actually inserting to them
echo Creating structure for Tätortskartan
for F in $GSD_PATH/tatort/0336/*.shp; do
    TABLE_NAME=tatort_`basename $F .shp | cut -d_ -f1`
    /usr/lib/postgresql/9.1/bin/shp2pgsql -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Tätortskartan
for F in `find $GSD_PATH/tatort/0336/ -name "*.shp"`; do
    TABLE_NAME=tatort_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    /usr/lib/postgresql/9.1/bin/shp2pgsql -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Create tables for Vägkartan before actually inserting to them
echo Creating structure for Vägkartan
for F in $GSD_PATH/vagk/south/*.shp; do
    TABLE_NAME=vagk_`basename $F .shp | cut -d_ -f1`
    /usr/lib/postgresql/9.1/bin/shp2pgsql -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Vägkartan
FILES="`find $GSD_PATH/vagk/ -maxdepth 2 -name "*.shp" -not \( -name "tx_*.shp" -or -name "tl_*.shp" \)` `find $GSD_PATH/vagk/ -mindepth 3 \( -name "tx_*.shp" -or -name "tl_*.shp" \)`"
for F in $FILES; do
    TABLE_NAME=vagk_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    /usr/lib/postgresql/9.1/bin/shp2pgsql -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done<commit_msg>Broke out shp2pgsql command path to variable. Import all of Tätortskartan, not only 0336.<commit_after># Imports all GSD shape data to a database.
# NOTE: If the db exists, it is *dropped* before importing any data

DB=lmv
GSD_PATH=/media/local/MapData/lmv-shape
SQL_LOG=/dev/null
ERROR_LOG=gsd_import.log
SHP2PGSQL=/usr/lib/postgresql/9.1/bin/shp2pgsql

# Create a clean slate
rm $SQL_LOG
rm $ERROR_LOG
dropdb $DB

createdb -T template_postgis $DB

# Create tables for Terrängkartan before actually inserting to them
echo Creating structure for Terrängkartan
for F in $GSD_PATH/terrang/south/*.shp; do
    TABLE_NAME=terrang_`basename $F .shp | cut -d_ -f1`
    $SHP2PGSQL -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Terrängkartan, using text labels from gistext directory
FILES="`find $GSD_PATH/terrang/ -maxdepth 2 -name "*.shp" -not \( -name "tx_*.shp" -or -name "tl_*.shp" \)` `find $GSD_PATH/terrang/ -mindepth 3 \( -name "tx_*.shp" -or -name "tl_*.shp" \)`"
for F in $FILES; do
    TABLE_NAME=terrang_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    $SHP2PGSQL -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Create tables for Tätortskartan before actually inserting to them
echo Creating structure for Tätortskartan
for F in $GSD_PATH/tatort/0336/*.shp; do
    TABLE_NAME=tatort_`basename $F .shp | cut -d_ -f1`
    $SHP2PGSQL -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Tätortskartan
FILES="`find $GSD_PATH/tatort/ -maxdepth 2 -name "*.shp"`"
for F in $FILES; do
    TABLE_NAME=tatort_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    $SHP2PGSQL -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Create tables for Vägkartan before actually inserting to them
echo Creating structure for Vägkartan
for F in $GSD_PATH/vagk/south/*.shp; do
    TABLE_NAME=vagk_`basename $F .shp | cut -d_ -f1`
    $SHP2PGSQL -p -W latin1 -I $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done

# Actually import Vägkartan
FILES="`find $GSD_PATH/vagk/ -maxdepth 2 -name "*.shp" -not \( -name "tx_*.shp" -or -name "tl_*.shp" \)` `find $GSD_PATH/vagk/ -mindepth 3 \( -name "tx_*.shp" -or -name "tl_*.shp" \)`"
for F in $FILES; do
    TABLE_NAME=vagk_`basename $F .shp | cut -d_ -f1`
    echo Importing $F into $TABLE_NAME
    $SHP2PGSQL -a -W latin1 $F $TABLE_NAME 2>>$ERROR_LOG | tee -a $SQL_LOG | psql -d $DB 1>/dev/null 2>>$ERROR_LOG
done
{'fraction_non_alphanumeric': 0.13887864256731833, 'fraction_numerical': 0.02102545186278126, 'mean_word_length': 3.61531914893617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4246013', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2263, 'n_words': 658}