<commit_before>#!/bin/bash
# dbc exports as csv are required

locales="enUS koKR frFR deDE zhCN zhTW esES esMX ruRU"
index=0

for loc in $locales; do
  if [ $1 ] && [ $1 != $loc ]; then
    index=$(expr $index + 1)
    continue;
  fi

  if [ "$loc" = "ruRU" ]; then index=0; fi

  file="output/${loc}/zones.lua"

  echo "## $loc ##"
  if [ -f "DBC/AreaTable_${loc}.dbc.csv" ]; then
    echo "pfDB[\"zones\"][\"${loc}\"] = {" > $file

    tail -n +2 DBC/AreaTable_${loc}.dbc.csv | while read line; do
      id=$(echo $line | cut -d , -f 1)
      entry=$(echo $line | cut -d , -f $(expr 12 + $index))
      if cat DBC/WorldMapArea_${loc}.dbc.csv| cut -d , -f 3 | grep "^$id$" &> /dev/null; then
        echo "  [$id] = $entry," >> $file
      fi
    done

    echo "}" >> $file
  fi

  index=$(expr $index + 1)
done<commit_msg>toolbox: query all zones at once<commit_after>#!/bin/bash
# dbc exports as csv are required

locales="enUS koKR frFR deDE zhCN zhTW esES esMX ruRU"
index=0

echo -n "zoneDB: "
for loc in $locales; do
  if [ "$loc" = "ruRU" ]; then index=0; fi

  mkdir -p output/${loc}
  file="output/${loc}/zoneDB.lua"

  echo -n "$loc "
  if [ -f "DBC/AreaTable_${loc}.dbc.csv" ]; then
    echo "pfDB[\"zones\"][\"${loc}\"] = {" > $file

    tail -n +2 DBC/AreaTable_${loc}.dbc.csv | while read line; do
      id=$(echo $line | cut -d , -f 1)
      entry=$(echo $line | cut -d , -f $(expr 12 + $index))
      if cat DBC/WorldMapArea_${loc}.dbc.csv| cut -d , -f 3 | grep "^$id$" &> /dev/null; then
        echo "  [$id] = $entry," >> $file
      fi
    done

    echo "}" >> $file
  fi

  index=$(expr $index + 1)
done
echo
{'fraction_non_alphanumeric': 0.20098039215686275, 'fraction_numerical': 0.011642156862745098, 'mean_word_length': 2.55, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '607904', 'n_tokens_mistral': 691, 'n_tokens_neox': 650, 'n_words': 190}