<commit_before>#!/usr/bin/env python
#
#  utf8_tests.py:  testing the svn client's utf8 (i18n) handling
#
#  Subversion is a tool for revision control. 
#  See http://subversion.tigris.org for more information.
#    
# ====================================================================
# Copyright (c) 2000-2002 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

# General modules
import shutil, stat, string, sys, re, os.path, os

# Our testing module
import svntest
from svntest import wc

# (abbreviation)
Item = wc.StateItem

#--------------------------------------------------------------------
# Data

# Here's a filename and a log message which contain some high-ascii
# data.  In theory this data has different interpretations when
# converting from 2 different charsets into UTF-8.

i18n_filename = "b";

i18n_logmsg = "drieëntwintig keer was één keer teveel";


######################################################################
# Tests
#
#   Each test must return 0 on success or non-zero on failure.


def basic_utf8_conversion(sbox):
  "conversion of paths and logs to/from utf8"

  if sbox.build():
    return 1

  wc_dir = sbox.wc_dir

  # Create the new i18n file and schedule it for addition
  svntest.main.file_append(os.path.join(wc_dir, i18n_filename), "hi")
  svntest.main.run_svn(None, 'add', os.path.join(wc_dir, i18n_filename))

  # Set our environment's locale to ISO-8859-1
  os.putenv('LC_ALL', 'ISO-8859-1')

  outlines, inlines = svntest.main.run_svn(None, # no error expected
                                           'commit', '-m', i18n_logmsg)
  if errlines:
    return 1


  return 0

# Here's how the test should really work:

# 1. sh LC_ALL=ISO-8859-1 svn commit <filename> -m "<logmsg>"<commit_msg>drie\303\253ntwintig keer was \303\251\303\251n keer teveel<commit_after>#!/usr/bin/env python
#
#  utf8_tests.py:  testing the svn client's utf8 (i18n) handling
#
#  Subversion is a tool for revision control. 
#  See http://subversion.tigris.org for more information.
#    
# ====================================================================
# Copyright (c) 2000-2002 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

# General modules
import shutil, stat, string, sys, re, os.path, os

# Our testing module
import svntest
from svntest import wc

# (abbreviation)
Item = wc.StateItem

#--------------------------------------------------------------------
# Data

# Here's a filename and a log message which contain some high-ascii
# data.  In theory this data has different interpretations when
# converting from 2 different charsets into UTF-8.

i18n_filename = "b";

i18n_logmsg = "drieëntwintig keer was één keer teveel";


######################################################################
# Tests
#
#   Each test must return 0 on success or non-zero on failure.


def basic_utf8_conversion(sbox):
  "conversion of paths and logs to/from utf8"

  if sbox.build():
    return 1

  wc_dir = sbox.wc_dir

  # Set our environment's locale to ISO-8859-1
  os.putenv('LC_ALL', 'ISO-8859-1')

  # Create the new i18n file and schedule it for addition
  svntest.main.file_append(os.path.join(wc_dir, i18n_filename), "hi")
  svntest.main.run_svn(None, 'add', os.path.join(wc_dir, i18n_filename))

  outlines, inlines = svntest.main.run_svn(None, # no error expected
                                           'commit', '-m', i18n_logmsg)
  if errlines:
    return 1


  return 0

# Here's how the test should really work:

# 1. sh LC_ALL=ISO-8859-1 svn commit <filename> -m "<logmsg>"
{'fraction_non_alphanumeric': 0.21197941038839493, 'fraction_numerical': 0.0271408516612073, 'mean_word_length': 3.371165644171779, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3217790', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1256, 'n_words': 436}