<commit_before>#ifndef _GLIBMM_USTRING_H
#define _GLIBMM_USTRING_H

/* Copyright (C) 2002 The gtkmm Development Team
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <glibmmconfig.h>
#include <glibmm/unicode.h>
#include <glib.h>

#include <cstddef> // for std::size_t and optionally std::ptrdiff_t
#include <initializer_list>
#include <iosfwd>
#include <iterator>
#include <sstream>
#include <string>

namespace Glib
{

#ifndef DOXYGEN_SHOULD_SKIP_THIS
#ifndef GLIBMM_HAVE_STD_ITERATOR_TRAITS

template <class T>
struct IteratorTraits
{
  using iterator_category = typename T::iterator_category;
  using value_type = typename T::value_type;
  using difference_type = typename T::difference_type;
  using pointer = typename T::pointer;
  using reference = typename T::reference;
};

template <class T>
struct IteratorTraits<T*>
{
  using iterator_category = std::random_access_iterator_tag;
  using value_type = T;
  using difference_type = std::ptrdiff_t;
  using pointer = T*;
  using reference = T&;
};

template <class T>
struct IteratorTraits<const T*>
{
  using iterator_category = std::random_access_iterator_tag;
  using value_type = T;
  using difference_type = std::ptrdiff_t;
  using pointer = const T*;
  using reference = const T&;
};

#endif /* GLIBMM_HAVE_STD_ITERATOR_TRAITS */
#endif /* DOXYGEN_SHOULD_SKIP_THIS */

/** The iterator type of Glib::ustring.
 * Note this is not a random access iterator but a bidirectional one,
 * since all index operations need to iterate over the UTF-8 data.  Use
 * std::advance() to move to a certain position.  However, all of the
 * relational operators are available:
 * <tt>==&nbsp;!=&nbsp;<&nbsp;>&nbsp;<=&nbsp;>=</tt>
 *
 * A writeable iterator isn't provided because:  The number of bytes of
 * the old UTF-8 character and the new one to write could be different.
 * Therefore, any write operation would invalidate all other iterators
 * pointing into the same string.
 *
 * The Glib::ustring iterated over must contain only valid UTF-8 data.
 * If it does not, operator++(), operator-\-() and operator*() may make
 * accesses outside the bounds of the string. A loop such as the following
 * one would not stop at the end of the string.
 * @code
 * // Bad code! Don't do this!
 * const char not_utf8[] = { '\x80', '\xef', '\x80', '\x80', '\xef', '\x80' };
 * const Glib::ustring s(not_utf8, not_utf8 + sizeof not_utf8);
 * for (Glib::ustring::const_iterator it = s.begin(); it != s.end(); ++it)
 *   std::cout << *it << std::endl;
 * @endcode
 *
 * @tparam T std::string::iterator or std::string::const_iterator
 */
template <class T>
class ustring_Iterator
{
public:
  using iterator_category = std::bidirectional_iterator_tag;
  using value_type = gunichar;
  using difference_type = std::string::difference_type;
  using reference = value_type;
  using pointer = void;

  inline ustring_Iterator();
  inline ustring_Iterator(const ustring_Iterator<std::string::iterator>& other);

  inline value_type operator*() const;

  inline ustring_Iterator<T>& operator++();
  inline const ustring_Iterator<T> operator++(int);
  inline ustring_Iterator<T>& operator--();
  inline const ustring_Iterator<T> operator--(int);

  explicit inline ustring_Iterator(T pos);
  inline T base() const;

private:
  T pos_;
};

/** Extract a UCS-4 character from UTF-8 data.
 * Convert a single UTF-8 (multibyte) character starting at @p pos to
 * a UCS-4 wide character.  This may read up to 6 bytes after the start
 * position, depending on the UTF-8 character width.  You have to make
 * sure the source contains at least one valid UTF-8 character.
 *
 * This is mainly used by the implementation of Glib::ustring::iterator,
 * but it might be useful as utility function if you prefer using
 * std::string even for UTF-8 encoding.
 */
gunichar get_unichar_from_std_iterator(std::string::const_iterator pos) G_GNUC_PURE;

/** %Glib::ustring has much the same interface as std::string, but contains
 * %Unicode characters encoded as UTF-8.<commit_msg>Glib::ustring_Iterator: Explicitly declare copy assignment<commit_after>#ifndef _GLIBMM_USTRING_H
#define _GLIBMM_USTRING_H

/* Copyright (C) 2002 The gtkmm Development Team
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <glibmmconfig.h>
#include <glibmm/unicode.h>
#include <glib.h>

#include <cstddef> // for std::size_t and optionally std::ptrdiff_t
#include <initializer_list>
#include <iosfwd>
#include <iterator>
#include <sstream>
#include <string>

namespace Glib
{

#ifndef DOXYGEN_SHOULD_SKIP_THIS
#ifndef GLIBMM_HAVE_STD_ITERATOR_TRAITS

template <class T>
struct IteratorTraits
{
  using iterator_category = typename T::iterator_category;
  using value_type = typename T::value_type;
  using difference_type = typename T::difference_type;
  using pointer = typename T::pointer;
  using reference = typename T::reference;
};

template <class T>
struct IteratorTraits<T*>
{
  using iterator_category = std::random_access_iterator_tag;
  using value_type = T;
  using difference_type = std::ptrdiff_t;
  using pointer = T*;
  using reference = T&;
};

template <class T>
struct IteratorTraits<const T*>
{
  using iterator_category = std::random_access_iterator_tag;
  using value_type = T;
  using difference_type = std::ptrdiff_t;
  using pointer = const T*;
  using reference = const T&;
};

#endif /* GLIBMM_HAVE_STD_ITERATOR_TRAITS */
#endif /* DOXYGEN_SHOULD_SKIP_THIS */

/** The iterator type of Glib::ustring.
 * Note this is not a random access iterator but a bidirectional one,
 * since all index operations need to iterate over the UTF-8 data.  Use
 * std::advance() to move to a certain position.  However, all of the
 * relational operators are available:
 * <tt>==&nbsp;!=&nbsp;<&nbsp;>&nbsp;<=&nbsp;>=</tt>
 *
 * A writeable iterator isn't provided because:  The number of bytes of
 * the old UTF-8 character and the new one to write could be different.
 * Therefore, any write operation would invalidate all other iterators
 * pointing into the same string.
 *
 * The Glib::ustring iterated over must contain only valid UTF-8 data.
 * If it does not, operator++(), operator-\-() and operator*() may make
 * accesses outside the bounds of the string. A loop such as the following
 * one would not stop at the end of the string.
 * @code
 * // Bad code! Don't do this!
 * const char not_utf8[] = { '\x80', '\xef', '\x80', '\x80', '\xef', '\x80' };
 * const Glib::ustring s(not_utf8, not_utf8 + sizeof not_utf8);
 * for (Glib::ustring::const_iterator it = s.begin(); it != s.end(); ++it)
 *   std::cout << *it << std::endl;
 * @endcode
 *
 * @tparam T std::string::iterator or std::string::const_iterator
 */
template <class T>
class ustring_Iterator
{
public:
  using iterator_category = std::bidirectional_iterator_tag;
  using value_type = gunichar;
  using difference_type = std::string::difference_type;
  using reference = value_type;
  using pointer = void;

  inline ustring_Iterator();
  inline ustring_Iterator(const ustring_Iterator<std::string::iterator>& other);
  ustring_Iterator& operator=(const ustring_Iterator& other) = default;

  inline value_type operator*() const;

  inline ustring_Iterator<T>& operator++();
  inline const ustring_Iterator<T> operator++(int);
  inline ustring_Iterator<T>& operator--();
  inline const ustring_Iterator<T> operator--(int);

  explicit inline ustring_Iterator(T pos);
  inline T base() const;

private:
  T pos_;
};

/** Extract a UCS-4 character from UTF-8 data.
 * Convert a single UTF-8 (multibyte) character starting at @p pos to
 * a UCS-4 wide character.  This may read up to 6 bytes after the start
 * position, depending on the UTF-8 character width.  You have to make
 * sure the source contains at least one valid UTF-8 character.
 *
 * This is mainly used by the implementation of Glib::ustring::iterator,
 * but it might be useful as utility function if you prefer using
 * std::string even for UTF-8 encoding.
 */
gunichar get_unichar_from_std_iterator(std::string::const_iterator pos) G_GNUC_PURE;

/** %Glib::ustring has much the same interface as std::string, but contains
 * %Unicode characters encoded as UTF-8.
{'fraction_non_alphanumeric': 0.10301802282179802, 'fraction_numerical': 0.006398634957875653, 'mean_word_length': 4.578822129684712, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2319156', 'n_tokens_mistral': 2915, 'n_tokens_neox': 2831, 'n_words': 1070}