<commit_before># Extract authority synonyms from translator database and create synonym
# files usable in Solr and Elasticsearch
#!/bin/bash

config_file="/usr/local/var/lib/tuelib/translations.conf"
db=$(inifile_lookup ${config_file} "Database" "sql_username")
db_user=$(inifile_lookup ${config_file} "Database" "sql_username")
db_password=$(inifile_lookup ${config_file} "Database" "sql_password")
outdir="/tmp/synonyms"

mkdir -p ${outdir}

declare -A langmap=(["eng"]="en" ["fre"]="fr" ["ger"]="de" ["gre"]="el"  ["ita"]="it"  ["por"]="pt" ["rus"]="ru" ["spa"]="es")

function GetGermanSynonyms {
    lang='ger'
    sshpass -p ${db_password} mysql --user=${db_user} --password --skip-column-names ${db} <<EOF
    SELECT GROUP_CONCAT(translation) FROM keyword_translations WHERE language_code='${lang}'
    AND gnd_code IN (SELECT gnd_code FROM keyword_translations WHERE language_code='${lang}' AND status='reliable') GROUP BY gnd_code;
EOF
}


function GetSynonymsForLanguage {
   lang=${1}
   sshpass -p "${db_password}" mysql --user="${db_user}" --password --skip-column-names ${db} <<EOF
   SELECT GROUP_CONCAT(translation) FROM keyword_translations WHERE language_code='${lang}' GROUP BY gnd_code;
EOF
}


function GetAllSynonyms {
   sshpass -p "${db_password}" mysql --user="${db_user}" --password --skip-column-names ${db} <<EOF
   SELECT GROUP_CONCAT(translation) FROM keyword_translations GROUP BY gnd_code;
EOF
}


function CleanUp {
   file=${1}
   # Remove specifications that most probably do not occur in fulltext
   sed --in-place --regexp-extended --expression 's/[[:space:]]*<.*>[[:space:]]*//g' $file
   # Remove erroneous newline/tab sequences
   sed --in-place  --regexp-extended --expression 's/(\\n|\\t)+//g' $file
   # Replace synonym separator #
   sed --in-place --regexp-extended --expression 's/#/,/g' $file
}

#German synonyms
outfile_de=${outdir}/synonyms_de.txt
echo "Creating ${outfile_de}"
GetGermanSynonyms > ${outfile_de}


#Synonyms for translated languages
for lang in "eng" "fre" "gre" "ita" "por" "rus" "spa"; do
   outfile=${outdir}/synonyms_${langmap[$lang]}
   echo "Creating ${outfile}"
   GetSynonymsForLanguage $lang > ${outfile}
   CleanUp ${outfile}
done

#All Synonyms
outfile_all="${outdir}/synonyms_all.txt"
echo "Creating ${outfile_all}"
GetAllSynonyms > ${outfile_all}
CleanUp ${outfile_all}



<commit_msg>Minor fixes and improvements<commit_after># Extract authority synonyms from translator database and create synonym
# files usable in Solr and Elasticsearch
#!/bin/bash

config_file="/usr/local/var/lib/tuelib/translations.conf"
db=$(inifile_lookup ${config_file} "Database" "sql_username")
db_user=$(inifile_lookup ${config_file} "Database" "sql_username")
db_password=$(inifile_lookup ${config_file} "Database" "sql_password")
outdir="/tmp/synonyms"

mkdir -p ${outdir}

declare -A langmap=(["eng"]="en" ["fre"]="fr" ["ger"]="de" ["gre"]="el"  ["ita"]="it"  ["por"]="pt" ["rus"]="ru" ["spa"]="es")

function GetGermanSynonyms {
    lang='ger'
    sshpass -p ${db_password} mysql --user=${db_user} --password --skip-column-names ${db} <<EOF
    SELECT GROUP_CONCAT(translation) FROM keyword_translations WHERE language_code='${lang}'
    AND gnd_code IN (SELECT gnd_code FROM keyword_translations WHERE language_code='${lang}' AND status='reliable') GROUP BY gnd_code;
EOF
}


function GetSynonymsForLanguage {
   lang=${1}
   sshpass -p "${db_password}" mysql --user="${db_user}" --password --skip-column-names ${db} <<EOF
   SELECT GROUP_CONCAT(translation) FROM keyword_translations WHERE language_code='${lang}' GROUP BY gnd_code;
EOF
}


function GetAllSynonyms {
   sshpass -p "${db_password}" mysql --user="${db_user}" --password --skip-column-names ${db} <<EOF
   SELECT GROUP_CONCAT(translation) FROM keyword_translations GROUP BY gnd_code;
EOF
}


function CleanUp {
   file=${1}
   # Remove specifications that most probably do not occur in fulltext
   sed --in-place --regexp-extended --expression 's/[[:space:]]*<[^>]+>[[:space:]]*//g' $file
   # Remove erroneous newline/tab sequences
   sed --in-place  --regexp-extended --expression 's/(\\n|\\t)+//g' $file
   # Replace synonym separator #
   sed --in-place --regexp-extended --expression 's/#/, /g' $file
}

#German synonyms
outfile_de=${outdir}/synonyms_de.txt
echo "Creating ${outfile_de}"
GetGermanSynonyms > ${outfile_de}


#Synonyms for translated languages
for lang in "eng" "fre" "gre" "ita" "por" "rus" "spa"; do
   outfile=${outdir}/synonyms_${langmap[$lang]}.txt
   echo "Creating ${outfile}"
   GetSynonymsForLanguage $lang > ${outfile}
   CleanUp ${outfile}
done

#All Synonyms
outfile_all="${outdir}/synonyms_all.txt"
echo "Creating ${outfile_all}"
GetAllSynonyms > ${outfile_all}
CleanUp ${outfile_all}




{'fraction_non_alphanumeric': 0.16719309328279638, 'fraction_numerical': 0.0008422825858075384, 'mean_word_length': 5.160830090791181, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '411344', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1601, 'n_words': 402}