<commit_before>base_dir=$1
config=$2
src_dir=$3
extracts_dir=$4
log=$5

# make sure only one is running at any time...
LOCK_FILE="${base_dir}/locks/mjolnir.lock"
mkdir -p "${base_dir}/locks"

(set -C; : > ${LOCK_FILE}) 2> /dev/null

if [ $? != "0" ]; then
   echo "Lock File exists - exiting"
   exit 1
fi

extracts=`find ${extracts_dir} -type f -name "*.pbf"`

tile_dir=`cat ${config} | jq '.mjolnir.hierarchy.tile_dir' | sed 's/^"\(.*\)"$/\1/'` || exit $?
mjolnir_tile_dir=$(echo ${tile_dir} | sed 's/tiles/mjolnir_tiles/g') || exit $?

sed -i 's/tiles/mjolnir_tiles/g' ${config} || exit $?

# clean mjolnir tiles
rm -rf ${mjolnir_tile_dir}/* || exit $?

export PATH=$PATH:/usr/local/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

# create admins
pbfadminbuilder -c ${config} ${extracts} || exit $?
<commit_msg>moved lua logic to scripts<commit_after>base_dir=$1
config=$2
src_dir=$3
extracts_dir=$4
log=$5

# make sure only one is running at any time...
LOCK_FILE="${base_dir}/locks/mjolnir.lock"
mkdir -p "${base_dir}/locks"

(set -C; : > ${LOCK_FILE}) 2> /dev/null

if [ $? != "0" ]; then
   echo "Lock File exists - exiting"
   exit 1
fi

conf_dir=$(dirname "${src_dir}/mjolnir/conf") || exit $?
config_dir=$(dirname "${src_dir}") || exit $?
lua_files=`find ${conf_dir} -type f -name "*.lua"` || exit $?

for file in ${lua_files} ; do
  file_name=`find ${file} -type f -name "*.lua" -printf '%f '` || exit $?
  ln -s ${conf_dir}/${file_name} ${config_dir}/${file} || exit $?
done

extracts=`find ${extracts_dir} -type f -name "*.pbf"`

tile_dir=`cat ${config} | jq '.mjolnir.hierarchy.tile_dir' | sed 's/^"\(.*\)"$/\1/'` || exit $?
mjolnir_tile_dir=$(echo ${tile_dir} | sed 's/tiles/mjolnir_tiles/g') || exit $?

sed -i 's/tiles/mjolnir_tiles/g' ${config} || exit $?

# clean mjolnir tiles
rm -rf ${mjolnir_tile_dir}/* || exit $?

export PATH=$PATH:/usr/local/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

# create admins
pbfadminbuilder -c ${config} ${extracts} || exit $?

{'fraction_non_alphanumeric': 0.20976581963129048, 'fraction_numerical': 0.008968609865470852, 'mean_word_length': 4.340425531914893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3685166', 'n_tokens_mistral': 875, 'n_tokens_neox': 815, 'n_words': 187}