<commit_before># /bin/bash

TOOLDIR=osm2po
DEST=ahb
DATAROOT="http://download.geofabrik.de/north-america/us/"
FILES=("california" "oregon" "washington" "idaho")

# grab tool
if [ ! -d "$TOOLDIR" ]; then
  wget http://osm2po.de/releases/osm2po-5.1.0.zip
  unzip osm2po-5.1.0.zip -d osm2po
  rm osm2po-5.1.0.zip
fi

# prepare each state
ALL=""
for i in "${FILES[@]}"
do
  java -Xmx1408m -jar osm2po/osm2po-core-5.1.0-signed.jar cmd=c workDir=data/$i prefix=$i tileSize=x,c $DATAROOT$i-latest.osm.pbf
  ALL=$ALL" "$i
done

# merge each state into $DEST.  fires up test server when completed
java -Xmx1g -jar osm2po/osm2po-core-5.1.0-signed.jar workDir=data/$DEST cmd=m tileSize=x,c prefix=$DEST $ALL<commit_msg>quick fix<commit_after># /bin/bash

TOOLDIR=osm2po
DEST=ahb
DATAROOT="http://download.geofabrik.de/north-america/us/"
FILES=("california" "oregon" "washington" "idaho")

# grab tool
if [ ! -d "$TOOLDIR" ]; then
  wget http://osm2po.de/releases/osm2po-5.1.0.zip
  unzip osm2po-5.1.0.zip -d osm2po
  rm osm2po-5.1.0.zip
fi

# prepare each state
ALL=""
for i in "${FILES[@]}"
do
  java -Xmx1408m -jar osm2po/osm2po-core-5.1.0-signed.jar cmd=c workDir=data/$i prefix=$i tileSize=x,c $DATAROOT$i-latest.osm.pbf
  ALL=$ALL" data/"$i
done

# merge each state into $DEST.  fires up test server when completed
java -Xmx1g -jar osm2po/osm2po-core-5.1.0-signed.jar workDir=data/$DEST cmd=m tileSize=x,c prefix=$DEST $ALL
{'fraction_non_alphanumeric': 0.16866619618913198, 'fraction_numerical': 0.04234297812279464, 'mean_word_length': 4.082437275985663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1041380', 'n_tokens_mistral': 662, 'n_tokens_neox': 631, 'n_words': 130}