<commit_before>#!/bin/sh

POT=app/locale/default.pot
POT_DIR=$(dirname $POT)

list_source_files() {
    find app/ -iname '*.ctp' -o -iname '*.php' | LC_ALL=C sort
}

throw_to_gettext() {
    xargs xgettext \
        --language=php --from-code=UTF-8 \
        --output=- --color=no --no-wrap \
        --keyword=__ --keyword=__p:1c,2
}

adjust_file_refs() {
    sed 's/#: \([^ ]\+\) \([^ ]\+\)/#: \1\n#: \2/g' | \
    sed 's/^#: \([^:]\+\):\([0-9]\+\)/#: github.com\/Tatoeba\/tatoeba2\/tree\/master\/\1#L\2/'
}

xgettext --version >/dev/null 2>&1 || {
    echo "Error: xgettext not available."
    exit 1
}

[ -d "$POT_DIR" ] || {
    echo "Error: can't find directory $POT_DIR from here."
         "Please move to the root directory of the repository."
    exit 1
}

list_source_files | throw_to_gettext | adjust_file_refs > "$POT" \
    && echo "$POT created." \
    || echo "Something went wrong."
<commit_msg>Extract domain-specific strings in custom shell.<commit_after>#!/bin/bash

POT=app/locale/default.pot
POT_DIR=$(dirname $POT)
POT_TMP=$(mktemp --suffix=.po)

# set the sighandlers asap
trap "rm -f $POT_TMP; exit" SIGHUP SIGINT SIGTERM

list_source_files() {
    find app/ -iname '*.ctp' -o -iname '*.php' | LC_ALL=C sort
}

throw_to_gettext() {
    xargs xgettext \
        --language=php --from-code=UTF-8 \
        --output=- --color=no --no-wrap \
        "$@"
}

adjust_file_refs() {
    sed 's/#: \([^ ]\+\) \([^ ]\+\)/#: \1\n#: \2/g' | \
    sed 's/^#: \([^:]\+\):\([0-9]\+\)/#: github.com\/Tatoeba\/tatoeba2\/tree\/master\/\1#L\2/'
}

get_all_contexts_from() {
    local pofile="$1"
    grep '^msgctxt' "$pofile" | sort | uniq | cut -d'"' -f2
}

write_domain_pot_from_context() {
    local domain="$1"
    local domain_potfile="$POT_DIR"/"$1".pot
    msggrep \
        --output=- --color=no \
        --msgctxt --fixed-strings --regexp="$domain" "$POT_TMP" | \
        grep -v '^msgctxt' > "$domain_potfile" \
        && echo "$domain_potfile created." \
        || echo "Something went wrong while creating $domain_potfile."
}

xgettext --version >/dev/null 2>&1 || {
    echo "Error: xgettext not available."
    exit 1
}

[ -d "$POT_DIR" ] || {
    echo "Error: can't find directory $POT_DIR from here."
    echo "Please move to the root directory of the repository."
    exit 1
}

# Extract strings to default.pot
DEFAULT_DOMAIN_K='--keyword=__ --keyword=__p:1c,2'
list_source_files | \
    throw_to_gettext $DEFAULT_DOMAIN_K | \
    adjust_file_refs > "$POT" \
    && echo "$POT created." \
    || echo "Something went wrong while creating $POT."

# Extract strings to domain-specific pot files.
#
# XGettext doesn't allow to easily extract strings to different files.
# To work around this problem, we first analyse domain-specific strings
# using the domain as msgctxt. Then, we split each contextualized string
# into a different file
OTHER_DOMAINS_K='--keyword=__d:1c,2'
list_source_files | \
    throw_to_gettext $OTHER_DOMAINS_K | \
    adjust_file_refs > "$POT_TMP"

# sounds like the charset is autodetected and the autodetection fails
sed -i 's/charset=CHARSET/charset=utf-8/' "$POT_TMP"

for domain in $(get_all_contexts_from "$POT_TMP"); do
    write_domain_pot_from_context "$domain"
done

rm -f $POT_TMP

{'fraction_non_alphanumeric': 0.17741437827830916, 'fraction_numerical': 0.01079913606911447, 'mean_word_length': 2.8779904306220097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1838744', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1225, 'n_words': 290}