<commit_before>/*
 * utf.c:  UTF-8 conversion routines
 *
 * ====================================================================
 * Copyright (c) 2000-2002 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 *
 * This software consists of voluntary contributions made by many
 * individuals.  For exact contribution history, see the revision
 * history and logs, available at http://subversion.tigris.org/.
 * ====================================================================
 */




#include <string.h>
#include <ctype.h>
#include <assert.h>

#include <apr_strings.h>
#include <apr_lib.h>
#include <apr_xlate.h>

#include "svn_string.h"
#include "svn_error.h"
#include "svn_pools.h"
#include "svn_utf.h"




#define SVN_UTF_NTOU_XLATE_HANDLE "svn-utf-ntou-xlate-handle"
#define SVN_UTF_UTON_XLATE_HANDLE "svn-utf-uton-xlate-handle"

/* Return the apr_xlate handle for converting native characters to UTF-8.
   Create one if it doesn't exist.  If unable to find a handle, or
   unable to create one because apr_xlate_open returned EINVAL, then
   set *RET to null and return SVN_NO_ERROR; if fail for some other
   reason, return error. */
static svn_error_t *
get_ntou_xlate_handle (apr_xlate_t **ret, apr_pool_t *pool)
{
  void *old_handle = NULL;
  apr_pool_t *parent, *global_pool;
  apr_status_t apr_err;

  /* ### I'm worried about the performance implications of searching
   * up the pool tree every time we call this function.  Leaving it in
   * for now, but if this turns out to be a bottleneck, then we should
   * store the xlation handles in some more quickly accessible place.
   *
   *    -kfogel,  7 July 2002
   */
  /* Find the global pool */
  for (global_pool = pool;
       (parent = apr_pool_get_parent (global_pool));
       global_pool = parent) ;

  /* If we already have a handle, just return it. */
  apr_pool_userdata_get (&old_handle, SVN_UTF_NTOU_XLATE_HANDLE, global_pool);
  if (old_handle != NULL) {
    *ret = old_handle;
    return SVN_NO_ERROR;
  }

  /* Try to create one. */
  apr_err = apr_xlate_open (ret, "UTF-8", APR_LOCALE_CHARSET, global_pool);

  /* apr_xlate_open returns EINVAL if no handle could be found. */
  if (apr_err == EINVAL)
    {
      *ret = NULL;
      return SVN_NO_ERROR;
    }
  else if (apr_err != APR_SUCCESS)
    return svn_error_create (apr_err, 0, NULL, pool,
                             "failed to create a converter to UTF-8");

  /* Save it for later. */
  apr_pool_userdata_set (*ret, SVN_UTF_NTOU_XLATE_HANDLE,
                         apr_pool_cleanup_null, global_pool);

  return SVN_NO_ERROR;
}


/* Return the apr_xlate handle for converting UTF-8 to native characters.
   Create one if it doesn't exist.  If unable to find a handle, or
   unable to create one because apr_xlate_open returned EINVAL, then
   set *RET to null and return SVN_NO_ERROR; if fail for some other
   reason, return error. */
static svn_error_t *
get_uton_xlate_handle (apr_xlate_t **ret, apr_pool_t *pool)
{
  void *old_handle = NULL;
  apr_pool_t *parent, *global_pool;
  apr_status_t apr_err;

  /* ### I'm worried about the performance implications of searching
   * up the pool tree every time we call this function.  Leaving it in
   * for now, but if this turns out to be a bottleneck, then we should
   * store the xlation handles in some more quickly accessible place.
   *
   *    -kfogel,  7 July 2002
   */
  /* Find the global pool */
  for (global_pool = pool;
       (parent = apr_pool_get_parent (global_pool));
       global_pool = parent) ;

  /* If we already have a handle, just return it. */
  apr_pool_userdata_get (&old_handle, SVN_UTF_UTON_XLATE_HANDLE, global_pool);
  if (old_handle != NULL) {
    *ret = old_handle;
    return SVN_NO_ERROR;
  }

  /* Try to create one. */
  apr_err = apr_xlate_open (ret, APR_LOCALE_CHARSET, "UTF-8", global_pool);

  /* apr_xlate_open returns EINVAL if no handle could be found. */
  if (apr_err == EINVAL)
    {
      *ret = NULL;
      return SVN_NO_ERROR;
    }
  if (apr_err != APR_SUCCESS)
    return svn_error_create (apr_err, 0, NULL, pool,
                             "failed to create a converter from UTF-8");

  /* Save it for later. */
  apr_pool_userdata_set (*ret, SVN_UTF_UTON_XLATE_HANDLE,
                         apr_pool_cleanup_null, global_pool);

  return SVN_NO_ERROR;
}


/* Convert SRC_LENGTH bytes of SRC_DATA in CONVSET, store the result
   in *DEST, which is allocated in POOL. */
static svn_error_t *
convert_to_stringbuf (apr_xlate_t *convset,
                      const char *src_data,
                      apr_size_t src_length,
                      svn_stringbuf_t **dest,
                      apr_pool_t *pool)
{
  apr_size_t buflen = src_length;
  apr_status_t apr_err;
  apr_size_t srclen = src_length;
  apr_size_t destlen = 0;<commit_msg>Extend the change committed in revision 2586:<commit_after>/*
 * utf.c:  UTF-8 conversion routines
 *
 * ====================================================================
 * Copyright (c) 2000-2002 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 *
 * This software consists of voluntary contributions made by many
 * individuals.  For exact contribution history, see the revision
 * history and logs, available at http://subversion.tigris.org/.
 * ====================================================================
 */




#include <string.h>
#include <ctype.h>
#include <assert.h>

#include <apr_strings.h>
#include <apr_lib.h>
#include <apr_xlate.h>

#include "svn_string.h"
#include "svn_error.h"
#include "svn_pools.h"
#include "svn_utf.h"




#define SVN_UTF_NTOU_XLATE_HANDLE "svn-utf-ntou-xlate-handle"
#define SVN_UTF_UTON_XLATE_HANDLE "svn-utf-uton-xlate-handle"

/* Return the apr_xlate handle for converting native characters to UTF-8.
   Create one if it doesn't exist.  If unable to find a handle, or
   unable to create one because apr_xlate_open returned APR_EINVAL, then
   set *RET to null and return SVN_NO_ERROR; if fail for some other
   reason, return error. */
static svn_error_t *
get_ntou_xlate_handle (apr_xlate_t **ret, apr_pool_t *pool)
{
  void *old_handle = NULL;
  apr_pool_t *parent, *global_pool;
  apr_status_t apr_err;

  /* ### I'm worried about the performance implications of searching
   * up the pool tree every time we call this function.  Leaving it in
   * for now, but if this turns out to be a bottleneck, then we should
   * store the xlation handles in some more quickly accessible place.
   *
   *    -kfogel,  7 July 2002
   */
  /* Find the global pool */
  for (global_pool = pool;
       (parent = apr_pool_get_parent (global_pool));
       global_pool = parent) ;

  /* If we already have a handle, just return it. */
  apr_pool_userdata_get (&old_handle, SVN_UTF_NTOU_XLATE_HANDLE, global_pool);
  if (old_handle != NULL) {
    *ret = old_handle;
    return SVN_NO_ERROR;
  }

  /* Try to create one. */
  apr_err = apr_xlate_open (ret, "UTF-8", APR_LOCALE_CHARSET, global_pool);

  if (APR_STATUS_IS_EINVAL (apr_err) || APR_STATUS_IS_ENOTIMPL (apr_err))
    {
      *ret = NULL;
      return SVN_NO_ERROR;
    }
  else if (apr_err != APR_SUCCESS)
    return svn_error_create (apr_err, 0, NULL, pool,
                             "failed to create a converter to UTF-8");

  /* Save it for later. */
  apr_pool_userdata_set (*ret, SVN_UTF_NTOU_XLATE_HANDLE,
                         apr_pool_cleanup_null, global_pool);

  return SVN_NO_ERROR;
}


/* Return the apr_xlate handle for converting UTF-8 to native characters.
   Create one if it doesn't exist.  If unable to find a handle, or
   unable to create one because apr_xlate_open returned APR_EINVAL, then
   set *RET to null and return SVN_NO_ERROR; if fail for some other
   reason, return error. */
static svn_error_t *
get_uton_xlate_handle (apr_xlate_t **ret, apr_pool_t *pool)
{
  void *old_handle = NULL;
  apr_pool_t *parent, *global_pool;
  apr_status_t apr_err;

  /* ### I'm worried about the performance implications of searching
   * up the pool tree every time we call this function.  Leaving it in
   * for now, but if this turns out to be a bottleneck, then we should
   * store the xlation handles in some more quickly accessible place.
   *
   *    -kfogel,  7 July 2002
   */
  /* Find the global pool */
  for (global_pool = pool;
       (parent = apr_pool_get_parent (global_pool));
       global_pool = parent) ;

  /* If we already have a handle, just return it. */
  apr_pool_userdata_get (&old_handle, SVN_UTF_UTON_XLATE_HANDLE, global_pool);
  if (old_handle != NULL) {
    *ret = old_handle;
    return SVN_NO_ERROR;
  }

  /* Try to create one. */
  apr_err = apr_xlate_open (ret, APR_LOCALE_CHARSET, "UTF-8", global_pool);

  if (APR_STATUS_IS_EINVAL (apr_err) || APR_STATUS_IS_ENOTIMPL (apr_err))
    {
      *ret = NULL;
      return SVN_NO_ERROR;
    }
  if (apr_err != APR_SUCCESS)
    return svn_error_create (apr_err, 0, NULL, pool,
                             "failed to create a converter from UTF-8");

  /* Save it for later. */
  apr_pool_userdata_set (*ret, SVN_UTF_UTON_XLATE_HANDLE,
                         apr_pool_cleanup_null, global_pool);

  return SVN_NO_ERROR;
}


/* Convert SRC_LENGTH bytes of SRC_DATA in CONVSET, store the result
   in *DEST, which is allocated in POOL. */
static svn_error_t *
convert_to_stringbuf (apr_xlate_t *convset,
                      const char *src_data,
                      apr_size_t src_length,
                      svn_stringbuf_t **dest,
                      apr_pool_t *pool)
{
  apr_size_t buflen = src_length;
  apr_status_t apr_err;
  apr_size_t srclen = src_length;
  apr_size_t destlen = 0;
{'fraction_non_alphanumeric': 0.10990409082012136, 'fraction_numerical': 0.006067723624975534, 'mean_word_length': 3.1863990167963947, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259460', 'n_tokens_mistral': 3511, 'n_tokens_neox': 3398, 'n_words': 1075}