<commit_before>#!/bin/bash
#
# loader update
# will go from buildout to grabbing proper otp.jar files
# rewritten February 2017
#
# install this in your crontab
# # run the loader each day at 2am and send the log to ~/loader.log
# 0 2 * * *  bash ~/update.sh link > ~/loader.log 2>&1

#
# loader update
# will go from buildout to grabbing proper otp.jar files
# June 2016
#

# set some variables
link=false
if [[ $1 == "link" ]]; then
  link=true
fi

start=$SECONDS
BHOME="/home/ubuntu"
LHOME="$BHOME/loader"
UHOME="$BHOME/utils"
GHOME="$BHOME/gtfsdb"

PROD_OTP_1="172.16.58.3"
PROD_OTP_2="172.16.58.3"
STG_OTP_1="172.16.31.10"


# function to print a log a message
log ()
{
  echo "--------------------------------------"
  printf "$1\n"
  echo "--------------------------------------"
}

# update the local GTFS files
cd /var/www/html/ ;
git clone https://gitlab.com/LACMTA/gtfs_lax.git ;
cd /var/www/html/gtfs_lax ;
git pull ;

if "$link"; then
  log "Great! we will link the big files."
else
  log "Redundant files ahead! consider this instead\nbash update.sh link"
fi

# upgrade the helper code
cd utils ; git pull

# make sure we have what's required
pip install setuptools==33.1.1 ;
pip install pyparsing  ;
pip install packaging ;
pip install zc.buildout ;

env buildout install prod

cd $GHOME ; git pull


env buildout install prod postgresql
cd $BHOME

log "OTT utils and gtfsdb installed"

# destroy the current loader
rm -Rf $LHOME

# checkout the latest loader code
git clone https://github.com/LACMTA/loader.git

# set up the virtualenv
cd $LHOME ;
virtualenv .

# run the buildout script
env buildout install lax

log "loader installed"

# install OSMOSIS if necessary
# OSMOSIS is the OpenStreetMap .pbf to .osm converter and db loader
cd ;
OSMBIN="$LHOME/ott/loader/osm/osmosis/bin/osmosis" ;
if [ -f "$OSMBIN" ];
then
    cd ;
    cd $LHOME/ott/loader/osm/osmosis/
    bash install.sh
    cd -
fi

log "Osmosis installed"

# install the protobuffer file and generate the OSM XML
cd ;
wget https://s3.amazonaws.com/metro-extracts.mapzen.com/los-angeles_california.osm.pbf \
  -O loader/ott/loader/otp/graph/lax/los-angeles_california.pbf ;

cd $BHOME ;
PBFILE="loader/ott/loader/otp/graph/lax/los-angeles_california.pbf"
if [ -f "$PBFILE" ];
then
  log "$PBFILE received"
fi

# remove the old OSM file
cd $BHOME ;
OSMFILE="$LHOME/ott/loader/otp/graph/lax/los-angeles_california.osm"
if [ -f "$OSMFILE" ];
then
    rm "$OSMFILE" ;
fi

cd $BHOME/ott/loader/otp/graph/lax/ ;
osmosis --read-pbf file=los-angeles_california.pbf --write-xml \
  los-angeles_california.osm

log "$osmfile file is ready"

# get the appropriate otp.jr file
# put it in place
cd $BHOME ;
JARFILE="$LHOME/ott/loader/otp/graph/lax/otp.jar"
if [ -f "$JARFILE" ];
then
    rm "$JARFILE" ;
fi

wget http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/1.0.0/otp-1.0.0-shaded.jar \
  -O "$JARFILE" ;

if [ -f "$JARFILE" ];
then
  log "otp.jar file installed"
  # print the otp headers
  cd $BHOME ;
  java -Xmx2G -jar loader/ott/loader/otp/graph/lax/otp.jar  --version
fi

log "let's grab the schedules and load them into the database"

# load GTFS schedules, OSM address data, and Bikeshare locations
cd $LHOME ;
bin/load_data -ini config/app.ini ;

# Generate the Graph.obj to introduce to the shaded otp.jar file
log "let's build the Graph.obj file"

cd $LHOME ;
bin/otp_build --no_tests lax  ;

log "ott/loader/otp/graph/lax/Graph.obj file is ready"

# copy the file to the otp servers
# NOTE: you need add a mechanism to update these IP addresses!
# NOTE: you need the public keys from the remote machines!
scp loader/ott/loader/otp/graph/lax/Graph.obj "$PROD_OTP_1:/tmp/Graph.obj"
scp loader/ott/loader/otp/graph/lax/Graph.obj "$PROD_OTP_2:/tmp/Graph.obj"
scp loader/ott/loader/otp/graph/lax/Graph.obj "$STG_OTP_1:/tmp/Graph.obj"

# do some remote voodoo to move the Graph.obj files into place
ssh -t $PROD_OTP_1 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;
  sudo -u root /etc/init.d/opentripplanner start ;
EOF

ssh -t $PROD_OTP_2 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;
  sudo -u root /etc/init.d/opentripplanner start ;
EOF

ssh -t $STG_OTP_1 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;<commit_msg>update variables<commit_after>#!/bin/bash
#
# loader update
# will go from buildout to grabbing proper otp.jar files
# rewritten February 2017
#
# install this in your crontab
# # run the loader each day at 2am and send the log to ~/loader.log
# 0 2 * * *  bash ~/update.sh link > ~/loader.log 2>&1

#
# loader update
# will go from buildout to grabbing proper otp.jar files
# June 2016
#

# set some variables
link=false
if [[ $1 == "link" ]]; then
  link=true
fi

start=$SECONDS
BHOME="/home/ubuntu"
LHOME="$BHOME/loader"
UHOME="$BHOME/utils"
GHOME="$BHOME/gtfsdb"

OSMBIN="$LHOME/ott/loader/osm/osmosis/bin/osmosis" ;
PBFILE="$LHOME/ott/loader/otp/graph/lax/los-angeles_california.pbf"
OSMFILE="$LHOME/ott/loader/otp/graph/lax/los-angeles_california.osm"
JARFILE="$LHOME/ott/loader/otp/graph/lax/otp.jar"

PROD_OTP_1="172.16.58.3"
PROD_OTP_2="172.16.58.3"
STG_OTP_1="172.16.31.10"


# function to print a log a message
log ()
{
  echo "--------------------------------------"
  printf "$1\n"
  echo "--------------------------------------"
}

# update the local GTFS files
cd /var/www/html/ ;
git clone https://gitlab.com/LACMTA/gtfs_lax.git ;
cd /var/www/html/gtfs_lax ;
git pull ;
# zip the agency schedule data
cd /var/www/html/gtfs_lax ; rm *.zip
find . -type d -maxdepth 1 -exec zip -r {}.zip {} \;


if "$link"; then
  log "Great! we will link the big files."
else
  log "Redundant files ahead! consider this instead\nbash update.sh link"
fi

# upgrade the helper code
cd $UHOME ; git pull

# make sure we have what's required
pip install setuptools==33.1.1 ;
pip install pyparsing  ;
pip install packaging ;
pip install zc.buildout ;

env buildout install prod

cd $GHOME ; git pull


env buildout install prod postgresql
cd $BHOME

log "OTT utils and gtfsdb installed"

# destroy the current loader
rm -Rf $LHOME

# checkout the latest loader code
git clone https://github.com/LACMTA/loader.git

# set up the virtualenv
cd $LHOME ;
virtualenv .

# run the buildout script
env buildout install lax

log "loader installed"

# install OSMOSIS if necessary
# OSMOSIS is the OpenStreetMap .pbf to .osm converter and db loader
if [ -f "$OSMBIN" ];
then
    cd $LHOME/ott/loader/osm/osmosis/
    bash install.sh
fi

log "Osmosis installed"

# install the protobuffer file and generate the OSM XML
wget https://s3.amazonaws.com/metro-extracts.mapzen.com/los-angeles_california.osm.pbf -O $PBFILE ;

if [ -f "$PBFILE" ];
then
  log "$PBFILE received"
fi

# remove the old OSM file
if [ -f "$OSMFILE" ];
then
    rm "$OSMFILE" ;
fi

cd $LHOME/ott/loader/otp/graph/lax/ ;
osmosis --read-pbf file=$PBFILE --write-xml $OSMFILE

# and to save 6.0Gb disk space:
cd $BHOME ;
if "$link"; then
  ln -s "$LHOME/ott/loader/otp/graph/lax/los-angeles_california.osm" "$LHOME/ott/loader/otp/graph/lax/los-angeles_california.osm" ;
else
    # fpurcell's code
    cp ../cache/osm/*.* ott/loader/osm/cache/
    sleep 5
    touch ott/loader/osm/cache/*.osm
fi

log "$osmfile file is ready"

# get the appropriate otp.jar file
# put it in place
cd $BHOME ;
if [ -f "$JARFILE" ];
then
    rm "$JARFILE" ;
fi

wget http://maven.conveyal.com.s3.amazonaws.com/org/opentripplanner/otp/1.0.0/otp-1.0.0-shaded.jar \
  -O "$JARFILE" ;

if [ -f "$JARFILE" ];
then
  log "otp.jar file installed"
  # print the otp headers
  cd $BHOME ;
  java -Xmx2G -jar $JARFILE  --version
fi

log "let's grab the schedules and load them into the database"

# load GTFS schedules, OSM address data, and Bikeshare locations
cd $LHOME ;
bin/load_data -ini config/app.ini ;

# Generate the Graph.obj to introduce to the shaded otp.jar file
log "let's build the Graph.obj file"

cd $LHOME ;
bin/otp_build --no_tests lax  ;

log "ott/loader/otp/graph/lax/Graph.obj file is ready"

# copy the file to the otp servers
# NOTE: you need add a mechanism to update these IP addresses!
# NOTE: you need the public keys from the remote machines!
scp $LHOME/ott/loader/otp/graph/lax/Graph.obj "$PROD_OTP_1:/tmp/Graph.obj"
scp $LHOME/ott/loader/otp/graph/lax/Graph.obj "$PROD_OTP_2:/tmp/Graph.obj"
scp $LHOME/ott/loader/otp/graph/lax/Graph.obj "$STG_OTP_1:/tmp/Graph.obj"

# do some remote voodoo to move the Graph.obj files into place
ssh -t $PROD_OTP_1 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;
  sudo -u root /etc/init.d/opentripplanner start ;
EOF

ssh -t $PROD_OTP_2 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;
  sudo -u root /etc/init.d/opentripplanner start ;
EOF

ssh -t $STG_OTP_1 << EOF
  sudo -u root mv /home/otp/graphs/lax/Graph.obj /tmp/Graph.obj.old ;
  sudo -u root chown otp:otp /tmp/Graph.obj ;
  sudo -u root mv /tmp/Graph.obj /home/otp/graphs/lax/Graph.obj ;
  sudo -u root ls -l /home/otp/graphs/lax/Graph.obj ;
  sudo -u root /etc/init.d/opentripplanner stop ;
{'fraction_non_alphanumeric': 0.14250638381457473, 'fraction_numerical': 0.012571204085641328, 'mean_word_length': 3.1444851444851443, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 5, 'xml': 2}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4116596', 'n_tokens_mistral': 4161, 'n_tokens_neox': 3983, 'n_words': 1107}