<commit_before># 
# (c) <NAME> 2002
#

# ToDo:
#   GHCi tests
#   expect failure for some ways only

import sys
import os
import string
import getopt
import platform
import time
import re

from testutil import *
from testglobals import *

# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'

# Try and find a utf8 locale to use
# First see if we already have a UTF8 locale
h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
v = h.read()
h.close()
if v == '':
    # We don't, so now see if 'locale -a' works
    h = os.popen('locale -a', 'r')
    v = h.read()
    h.close()
    if v != '':
        # If it does then use the first utf8 locale that is available
        h = os.popen('locale -a | grep -i utf8 2>/dev/null', 'r')
        v = h.readline().strip()
        h.close()
        if v != '':
            os.environ['LC_ALL'] = v<commit_msg>Fix setting of utf8 locale on Mac OS X<commit_after># 
# (c) <NAME> 2002
#

# ToDo:
#   GHCi tests
#   expect failure for some ways only

import sys
import os
import string
import getopt
import platform
import time
import re

from testutil import *
from testglobals import *

# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'

# Try and find a utf8 locale to use
# First see if we already have a UTF8 locale
h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
v = h.read()
h.close()
if v == '':
    # We don't, so now see if 'locale -a' works
    h = os.popen('locale -a', 'r')
    v = h.read()
    h.close()
    if v != '':
        # If it does then use the first utf8 locale that is available
        h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
        v = h.readline().strip()
        h.close()
        if v != '':
            os.environ['LC_ALL'] = v
{'fraction_non_alphanumeric': 0.12625754527162977, 'fraction_numerical': 0.013078470824949699, 'mean_word_length': 2.636197440585009, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3802565', 'n_tokens_mistral': 716, 'n_tokens_neox': 685, 'n_words': 277}