<commit_before>#!/bin/sh
# charset relation generator
#    $Id: gen_data.sh,v 1.8 2005/03/09 20:07:30 lav Exp $

OUTFILE=get_charset_data.h

print()
{
	printf " { \"%s\",\t%d,\t\"%s\",\t\"%s\",\t\"%s\",\t\"%s\" }%s\n" $@ >>$OUTFILE
}

echo Test string for $LANG locale:
./print_data_string || exit 1
echo
cat <<EOF >$OUTFILE
/* Do not edit this file!
   It is autogenerated from WINE program print_data_string.c
*/
static const struct charsetrel_entry charset_relation[] =
{
	/* locale,     lcid,    unix,        windows,    dos,      mac charset */
EOF
echo "Generating with WINE program..."
echo "This is log error file. See for your problem locale here and send me a mail: <EMAIL>">./gen_data.out.txt
for i in `locale -a | LC_ALL=C sort`
do
#	printf "\t{ %-15s %d,\t\"%s\",\t\"%s\",\t\"%s\",\t\"%s\" }" "\"$i\"," `LANG=$i ./print_data_string 2>/dev/null` >>$OUTFILE
	echo -e -n " {" >>$OUTFILE
	LANG=$i LC_CTYPE=$i ./print_data_string 2>>./gen_data.out.txt >>$OUTFILE
	echo -e " }," >>$OUTFILE
done
echo >>$OUTFILE
echo "/* Follow entries is dummy for ASCII/ANSI encoding */" >>$OUTFILE
print "POSIX" 1033 ASCII CP1252 IBM437 "MAC" ,
print "POSIX" 1033 ANSIX341968 CP1252 IBM437 "MAC"
cat <<EOF >>$OUTFILE

};
EOF<commit_msg>fix gen_data.sh script<commit_after>#!/bin/sh
# charset relation generator
#    $Id: gen_data.sh,v 1.8 2005/03/09 20:07:30 lav Exp $

OUTFILE=get_charset_data.h
PRINTDATAPROGRAM="wwo ./print_data_string.exe.so"

print()
{
	printf " { \"%s\",\t%d,\t\"%s\",\t\"%s\",\t\"%s\",\t\"%s\" }%s\n" $@ >>$OUTFILE
}

echo Test string for $LANG locale:
$PRINTDATAPROGRAM || exit 1
echo
cat <<EOF >$OUTFILE
/* Do not edit this file!
   It is autogenerated from WINE program print_data_string.c
*/
static const struct charsetrel_entry charset_relation[] =
{
	/* locale,     lcid,    unix,        windows,    dos,      mac charset */
EOF
echo "Generating with WINE program..."
echo "This is log error file. See for your problem locale here and send me a mail: <EMAIL>">./gen_data.out.txt
for i in `locale -a | LC_ALL=C sort`
do
	echo -e -n " {" >>$OUTFILE
	LANG=$i LC_CTYPE=$i $PRINTDATAPROGRAM 2>>./gen_data.out.txt >>$OUTFILE
	echo -e " }," >>$OUTFILE
done
echo >>$OUTFILE
echo "/* Follow entries is dummy for ASCII/ANSI encoding */" >>$OUTFILE
print "POSIX" 1033 ASCII CP1252 IBM437 "MAC" ,
print "POSIX" 1033 ANSIX341968 CP1252 IBM437 "MAC"
cat <<EOF >>$OUTFILE

};
EOF
{'fraction_non_alphanumeric': 0.17896060352053647, 'fraction_numerical': 0.039815590947191955, 'mean_word_length': 3.6349514563106795, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5483295', 'n_tokens_mistral': 1014, 'n_tokens_neox': 911, 'n_words': 268}