<commit_before>#!/bin/bash
################################################
# Builds whole planet in /media/ssd/common dir #
################################################

# At least "set -e -u" should always be here, not just for debugging!
# "set -x" is useful to see what is going on.
set -e -u -x

# global params
LIGHT_NODES=false
PROCESSORS=4

# displays usage and exits
function Usage {
  echo ''
  echo "Usage: $0 [path_to_data_folder_with_classsif_and_planet.osm.bz2] [bucketing_level] [optional_path_to_intermediate_data] [world_only]"
  echo "Note, that for coastlines bucketing_level should be 0"
  echo "Planet squares size is (2^bucketing_level x 2^bucketing_level)"
  echo "If optional intermediate path is given, only second pass will be executed"
  exit 0
}

# for parallel builds
function forky() {
  local num_par_procs
  if [[ -z $1 ]] ; then
    num_par_procs=2
  else
    num_par_procs=$1
  fi

  while [[ $(jobs | wc -l) -ge $num_par_procs ]] ; do
    sleep 1
  done
}

if [ $# -lt 2 ]; then
  Usage
fi

DATA_PATH=$1
BUCKETING_LEVEL=$2

# set up necessary Windows MinGW settings
#if [ ${WINDIR+1} ]; then
#fi

# check if we have QT in PATH
if [ ! `which qmake` ]; then
  echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:'
  echo '  1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH'
  echo '  2. Set it permanently by adding export... string above to your ~/.bashrc'
  echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc'
  exit 0
fi

# determine script path
MY_PATH=`dirname $0`

# find indexer_tool
IT_PATHS_ARRAY=(  "$MY_PATH/../../../omim-build-release/out/release/indexer_tool" \
                  "$MY_PATH/../../out/release/indexer_tool" )

for i in {0..1}; do
  if [ -x ${IT_PATHS_ARRAY[i]} ]; then
    INDEXER_TOOL=${IT_PATHS_ARRAY[i]}
    echo TOOL: $INDEXER_TOOL
    break
  fi
done

if [[ ! -n $INDEXER_TOOL ]]; then
  echo 'No indexer_tool found, please build omim-build-release or omim/out/release'
  echo ""
  Usage
fi

OSM_BZ2=$DATA_PATH/planet.osm.bz2

if ! [ -f $OSM_BZ2 ]; then
  echo "Can't open file $OSM_BZ2, did you forgot to specify dataDir?"
  echo ""
  Usage
fi

TMPDIR=$DATA_PATH/intermediate_data/

if [ $# -ge 3 ]; then
  TMPDIR=$3/
fi

if ! [ -d $TMPDIR ]; then
  mkdir -p $TMPDIR
fi

PV="cat"
if [ `which pv` ]
then
  PV=pv
fi

WORLD_ONLY=false
if [ $# -ge 4  ]; then
  WORLD_ONLY=true
fi


# skip 1st pass if intermediate data path was given
if [ $# -lt 3 ]; then
  # 1st pass - not paralleled
  $PV $OSM_BZ2 | bzip2 -d | $INDEXER_TOOL --intermediate_data_path=$TMPDIR \
    --use_light_nodes=$LIGHT_NODES \
    --preprocess_xml
fi

# 2nd pass - not paralleled
$PV $OSM_BZ2 | bzip2 -d | $INDEXER_TOOL --intermediate_data_path=$TMPDIR \
  --use_light_nodes=$LIGHT_NODES --bucketing_level=$BUCKETING_LEVEL \
  --generate_features --worldmap_max_zoom=5 --world_only=$WORLD_ONLY \
  --data_path=$DATA_PATH

# 3rd pass - do in parallel
for file in $DATA_PATH/*.mwm; do
  if [ $file != "minsk-pass"  ]; then
    filename=$(basename $file)
    extension=${filename##*.}
    filename=${filename%.*}
    $INDEXER_TOOL --data_path=$DATA_PATH --generate_geometry --sort_features --generate_index --output=$filename &
    forky $PROCESSORS
  fi
done

wait<commit_msg>Fixed files with spaces in build script<commit_after>#!/bin/bash
################################################
# Builds whole planet in /media/ssd/common dir #
################################################

# At least "set -e -u" should always be here, not just for debugging!
# "set -x" is useful to see what is going on.
set -e -u -x

# global params
LIGHT_NODES=false
PROCESSORS=4

# displays usage and exits
function Usage {
  echo ''
  echo "Usage: $0 [path_to_data_folder_with_classsif_and_planet.osm.bz2] [bucketing_level] [optional_path_to_intermediate_data] [world_only]"
  echo "Note, that for coastlines bucketing_level should be 0"
  echo "Planet squares size is (2^bucketing_level x 2^bucketing_level)"
  echo "If optional intermediate path is given, only second pass will be executed"
  exit 0
}

# for parallel builds
function forky() {
  local num_par_procs
  if [[ -z $1 ]] ; then
    num_par_procs=2
  else
    num_par_procs=$1
  fi

  while [[ $(jobs | wc -l) -ge $num_par_procs ]] ; do
    sleep 1
  done
}

if [ $# -lt 2 ]; then
  Usage
fi

DATA_PATH=$1
BUCKETING_LEVEL=$2

# set up necessary Windows MinGW settings
#if [ ${WINDIR+1} ]; then
#fi

# check if we have QT in PATH
if [ ! `which qmake` ]; then
  echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:'
  echo '  1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH'
  echo '  2. Set it permanently by adding export... string above to your ~/.bashrc'
  echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc'
  exit 0
fi

# determine script path
MY_PATH=`dirname $0`

# find indexer_tool
IT_PATHS_ARRAY=(  "$MY_PATH/../../../omim-build-release/out/release/indexer_tool" \
                  "$MY_PATH/../../out/release/indexer_tool" )

for i in {0..1}; do
  if [ -x ${IT_PATHS_ARRAY[i]} ]; then
    INDEXER_TOOL=${IT_PATHS_ARRAY[i]}
    echo TOOL: $INDEXER_TOOL
    break
  fi
done

if [[ ! -n $INDEXER_TOOL ]]; then
  echo 'No indexer_tool found, please build omim-build-release or omim/out/release'
  echo ""
  Usage
fi

OSM_BZ2=$DATA_PATH/planet.osm.bz2

if ! [ -f $OSM_BZ2 ]; then
  echo "Can't open file $OSM_BZ2, did you forgot to specify dataDir?"
  echo ""
  Usage
fi

TMPDIR=$DATA_PATH/intermediate_data/

if [ $# -ge 3 ]; then
  TMPDIR=$3/
fi

if ! [ -d $TMPDIR ]; then
  mkdir -p $TMPDIR
fi

PV="cat"
if [ `which pv` ]
then
  PV=pv
fi

WORLD_ONLY=false
if [ $# -ge 4  ]; then
  WORLD_ONLY=true
fi


# skip 1st pass if intermediate data path was given
if [ $# -lt 3 ]; then
  # 1st pass - not paralleled
  $PV $OSM_BZ2 | bzip2 -d | $INDEXER_TOOL --intermediate_data_path=$TMPDIR \
    --use_light_nodes=$LIGHT_NODES \
    --preprocess_xml
fi

# 2nd pass - not paralleled
$PV $OSM_BZ2 | bzip2 -d | $INDEXER_TOOL --intermediate_data_path=$TMPDIR \
  --use_light_nodes=$LIGHT_NODES --bucketing_level=$BUCKETING_LEVEL \
  --generate_features --worldmap_max_zoom=5 --world_only=$WORLD_ONLY \
  --data_path=$DATA_PATH

# 3rd pass - do in parallel
for file in $DATA_PATH/*.mwm; do
  if [ "$file" != "minsk-pass"  ]; then
    filename=$(basename "$file")
    extension="${filename##*.}"
    filename="${filename%.*}"
    $INDEXER_TOOL -output="$filename" -data_path=$DATA_PATH -generate_geometry -sort_features -generate_index -intermediate_data_path=$TMPDIR &
    forky $PROCESSORS
  fi
done

wait
{'fraction_non_alphanumeric': 0.14895848943503673, 'fraction_numerical': 0.011688895549228234, 'mean_word_length': 3.552523874488404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7038525', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2429, 'n_words': 716}