<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ZORBA_UNICODE_UTIL_H
#define ZORBA_UNICODE_UTIL_H

#include <zorba/config.h>

#ifndef ZORBA_NO_UNICODE

#include <cctype>
#include <cstring>
#include <cwchar>
#include <unicode/unistr.h>

namespace zorba {
namespace unicode {

////////// types //////////////////////////////////////////////////////////////

/**
 * The character type that can hold a Unicode character encoded in UTF-16.  Do
 * not assume that this is an unsigned type.
 */
typedef UChar char_type;

/**
 * The type type that can hold a Unicode code-point.  Do not assume that this
 * is an unsigned type.
 */
typedef UChar32 code_point;

typedef int32_t size_type;

/**
 * Unicode normalization modes.
 */
namespace normalization {
  enum type {
    none, ///< No decomposition/composition.
    NFC,  ///< Canonical decomposition followed by canonical composition.
    NFD,  ///< Canonical decomposition.
    NFKC, ///< Compatibility decomposition followed by canonical composition.
    NFKD  ///< Compatibility decomposition.
  };
}

/**
 * A Unicode string.
 */
typedef U_NAMESPACE_QUALIFIER UnicodeString string;

////////// code-point checking ////////////////////////////////////////////////

/**
 * Test whether the given character is invalid in an IRI.
 *
 * @param c The character.
 * @return Returns \c true only if the character is invalid in an IRI.
 * See RFC 3987.
 */
ZORBA_DLL_PUBLIC
bool is_invalid_in_iri( code_point c );

/**
 * Test whether the given character is a "iprivate".
 *
 * @param c The character.
 * @return Returns \c true only if the character is a "iprivate".
 * See RFC 3987.
 */
ZORBA_DLL_PUBLIC
bool is_iprivate( code_point c );

/**
 * Unicode version is isspace(3).
 *
 * @param c The code-point to check.
 * @return Returns \c true only if \a c is a whitespace character.
 */
inline bool is_space( code_point c ) {
  char ascii_c = static_cast<char>( c );
#ifdef WIN32
  // Windows' isspace() implementation crashes for non-ASCII characters.
  ascii_c &= 0x7F;
#endif
  return ascii_c == c && isspace( ascii_c );
}

/**
 * Test whether the given character is a "ucschar".
 *
 * @param c The character.
 * @return Returns \c true only if the character is a "ucschar".
 * See RFC 3987.<commit_msg>Better implementation os unicode::is_space().<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ZORBA_UNICODE_UTIL_H
#define ZORBA_UNICODE_UTIL_H

#include <zorba/config.h>

#ifndef ZORBA_NO_UNICODE

#include <cctype>
#include <cstring>
#include <cwchar>
#include <unicode/unistr.h>

namespace zorba {
namespace unicode {

////////// types //////////////////////////////////////////////////////////////

/**
 * The character type that can hold a Unicode character encoded in UTF-16.  Do
 * not assume that this is an unsigned type.
 */
typedef UChar char_type;

/**
 * The type type that can hold a Unicode code-point.  Do not assume that this
 * is an unsigned type.
 */
typedef UChar32 code_point;

typedef int32_t size_type;

/**
 * Unicode normalization modes.
 */
namespace normalization {
  enum type {
    none, ///< No decomposition/composition.
    NFC,  ///< Canonical decomposition followed by canonical composition.
    NFD,  ///< Canonical decomposition.
    NFKC, ///< Compatibility decomposition followed by canonical composition.
    NFKD  ///< Compatibility decomposition.
  };
}

/**
 * A Unicode string.
 */
typedef U_NAMESPACE_QUALIFIER UnicodeString string;

////////// code-point checking ////////////////////////////////////////////////

/**
 * Test whether the given character is invalid in an IRI.
 *
 * @param c The character.
 * @return Returns \c true only if the character is invalid in an IRI.
 * See RFC 3987.
 */
ZORBA_DLL_PUBLIC
bool is_invalid_in_iri( code_point c );

/**
 * Test whether the given character is a "iprivate".
 *
 * @param c The character.
 * @return Returns \c true only if the character is a "iprivate".
 * See RFC 3987.
 */
ZORBA_DLL_PUBLIC
bool is_iprivate( code_point c );

/**
 * Unicode version is isspace(3).
 *
 * @param c The code-point to check.
 * @return Returns \c true only if \a c is a whitespace character.
 */
inline bool is_space( code_point c ) {
#ifdef WIN32
  // Windows' isspace() implementation crashes for non-ASCII characters.
  char const ascii_c = static_cast<char>( c & 0x7F );
  return ascii_c == c && isspace( ascii_c );
#else
  return isspace( c );
#endif
}

/**
 * Test whether the given character is a "ucschar".
 *
 * @param c The character.
 * @return Returns \c true only if the character is a "ucschar".
 * See RFC 3987.
{'fraction_non_alphanumeric': 0.1357679538703477, 'fraction_numerical': 0.012231347195526822, 'mean_word_length': 2.8728010825439783, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2608333', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1649, 'n_words': 670}