<commit_before># Export from one main database into a game-specific one, importing only those keys listed as used in usedkeys.txt
# $1, parameter 1: The input folder containing the main database
# $2, parameter 2: The destination folder to export to (will start by removing existing localizations)
# $3, parameter 3: A text file containing exactly the keys to export, each on a seperate line

# The empty line above is important, it distinguishes between the usage message (printed) and the code
# If no parameters, print help atop this shell file
if [ -z $1 ]; then
	sed -e '/^$/,$d' $0;
	exit 1;
fi

LOCALIZATIONS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
WORKING_DIR=`pwd`;
cd $LOCALIZATIONS_DIR;
mkdir -p $WORKING_DIR/.internals;
git fetch;
git status | sed -n 's/HEAD detached/Warning: Localizations repository is on a detached head (you should check out master)/p' > $WORKING_DIR/.internals/gitstatusdiff.txt;
git status | grep "Changes not staged for commit" >> $WORKING_DIR/.internals/gitstatusdiff.txt;
git log HEAD..origin/master >> $WORKING_DIR/.internals/gitstatusdiff.txt;
GITSTATUS_LINES=`wc -l $WORKING_DIR/.internals/gitstatusdiff.txt | tr -s ' ' | cut -d' ' -f1`;
if [[ $GITSTATUS_LINES -gt 0 ]]; then
	echo "Your git status in the localizations repository, located at $LOCALIZATIONS_DIR is behind / dirty / detached."
	echo "It should be on branch master and up-to-date with origin/master.";
	echo "See .internals/gitstatusdiff.txt for exactly what the problems are.";
else
	rm -f $WORKING_DIR/.internals/gitstatusdiff.txt;
fi
cd $WORKING_DIR;

# in all scripts, if a folder is passed with trailing /, remove it and continue
inputFolder=`echo $1 | sed 's/\/$//'`;
destinationFolder=`echo $2 | sed 's/\/$//'`;

source util.sh
source langlist.sh

# turn lines containing e.g. "gold" into "^gold=", so that only exact matches are exported
sed 's/^/\^/' $3 | sed 's/$/=/' > .internals/keyregexes.txt

# make destination if it doesn't exist
mkdir -p $destinationFolder
# remove all files from it
for language in "${ALL_LANGUAGES[@]}"; do
	rm $destinationFolder/$language.txt
	grep -f .internals/keyregexes.txt $inputFolder/$language.txt > $destinationFolder/$language.txt<commit_msg>fix; export to empty dir without warnings<commit_after># Export from one main database into a game-specific one, importing only those keys listed as used in usedkeys.txt
# $1, parameter 1: The input folder containing the main database
# $2, parameter 2: The destination folder to export to (will start by removing existing localizations)
# $3, parameter 3: A text file containing exactly the keys to export, each on a seperate line

# The empty line above is important, it distinguishes between the usage message (printed) and the code
# If no parameters, print help atop this shell file
if [ -z $1 ]; then
	sed -e '/^$/,$d' $0;
	exit 1;
fi

LOCALIZATIONS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
WORKING_DIR=`pwd`;
cd $LOCALIZATIONS_DIR;
mkdir -p $WORKING_DIR/.internals;
git fetch;
git status | sed -n 's/HEAD detached/Warning: Localizations repository is on a detached head (you should check out master)/p' > $WORKING_DIR/.internals/gitstatusdiff.txt;
git status | grep "Changes not staged for commit" >> $WORKING_DIR/.internals/gitstatusdiff.txt;
git log HEAD..origin/master >> $WORKING_DIR/.internals/gitstatusdiff.txt;
GITSTATUS_LINES=`wc -l $WORKING_DIR/.internals/gitstatusdiff.txt | tr -s ' ' | cut -d' ' -f1`;
if [[ $GITSTATUS_LINES -gt 0 ]]; then
	echo "Your git status in the localizations repository, located at $LOCALIZATIONS_DIR is behind / dirty / detached."
	echo "It should be on branch master and up-to-date with origin/master.";
	echo "See .internals/gitstatusdiff.txt for exactly what the problems are.";
else
	rm -f $WORKING_DIR/.internals/gitstatusdiff.txt;
fi
cd $WORKING_DIR;

# in all scripts, if a folder is passed with trailing /, remove it and continue
inputFolder=`echo $1 | sed 's/\/$//'`;
destinationFolder=`echo $2 | sed 's/\/$//'`;

source util.sh
source langlist.sh

# turn lines containing e.g. "gold" into "^gold=", so that only exact matches are exported
sed 's/^/\^/' $3 | sed 's/$/=/' > .internals/keyregexes.txt

# make destination if it doesn't exist
mkdir -p $destinationFolder
# remove all files from it
for language in "${ALL_LANGUAGES[@]}"; do
	rm -f $destinationFolder/$language.txt
	grep -f .internals/keyregexes.txt $inputFolder/$language.txt > $destinationFolder/$language.txt
{'fraction_non_alphanumeric': 0.12126656186840333, 'fraction_numerical': 0.0067370312149112955, 'mean_word_length': 4.425091352009744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4567391', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1369, 'n_words': 544}