<commit_before>/* mtext.c -- M-text module.
   Copyright (C) 2003, 2004
     National Institute of Advanced Industrial Science and Technology (AIST)
     Registration Number H15PRO112

   This file is part of the m17n library.

   The m17n library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public License
   as published by the Free Software Foundation; either version 2.1 of
   the License, or (at your option) any later version.

   The m17n library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the m17n library; if not, write to the Free
   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   02111-1307, USA.  */

/***en
    @addtogroup m17nMtext
    @brief M-text objects and API for them.

    In the m17n library, text is represented as an object called @e
    M-text rather than as a C-string (<tt>char *</tt> or <tt>unsigned
    char *</tt>).  An M-text is a sequence of characters whose length
    is equals to or more than 0, and can be coined from various
    character sources, e.g. C-strings, files, character codes, etc.

    M-texts are more useful than C-strings in the following points.

    @li M-texts can handle mixture of characters of various scripts,
    including all Unicode characters and more.  This is an
    indispensable facility when handling multilingual text.

    @li Each character in an M-text can have properties called @e text
    @e properties. Text properties store various kinds of information
    attached to parts of an M-text to provide application programs
    with a unified view of those information.  As rich information can
    be stored in M-texts in the form of text properties, functions in
    application programs can be simple.

    In addition, the library provides many functions to manipulate an
    M-text just the same way as a C-string.  */

/***ja
    @addtogroup m17nMtext

    @brief M-text ֥ȤȤ˴ؤ API.

    m17n 饤֥ϡ C-string<tt>char *</tt>  <tt>unsigned
    char *</tt>ˤǤϤʤ @e M-text ȸƤ֥֥ȤǥƥȤɽ롣
    M-text Ĺ 0 ʾʸǤꡢʸʤȤ 
    C-stringե롢ʸˤǤ롣

    M-text ˤϡC-string ˤʤʲħ롣     

    @li M-text ¿μʸƱˡߤơƱ
    ȤǤ롣Unicode ƤʸϤ󡢤¿ʸ
    ǰ롣¿ƥȤ򰷤ǤɬܤεǽǤ롣

    @li M-text γʸϡ@e ƥȥץѥƥ ȸƤФץѥƥ
    ĤȤǤ롣ƥȥץѥƥˤäơƥȤγ̤
    ؤ͡ʾ M-text ݻ뤳ȤǤ롣Τᡢ
    ξ򥢥ץꥱץŪ˰ȤǤ롣
    ޤM-text Τ˭٤ʾĤᡢץꥱץ
    γƴؿǲ뤳ȤǤ롣

    m17n 饤֥ϡ C-string 뤿󶡤
    δؿƱΤΤ M-text 뤿˥ݡȤƤ롣  */

/*=*/

#if !defined (FOR_DOXYGEN) || defined (DOXYGEN_INTERNAL_MODULE)
/*** @addtogroup m17nInternal
     @{ */

#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <locale.h>

#include "m17n.h"
#include "m17n-misc.h"
#include "internal.h"
#include "textprop.h"
#include "character.h"
#include "mtext.h"
#include "plist.h"

static M17NObjectArray mtext_table;

static MSymbol M_charbag;

#ifdef WORDS_BIGENDIAN
static enum MTextFormat default_utf_16 = MTEXT_FORMAT_UTF_16BE;
static enum MTextFormat default_utf_32 = MTEXT_FORMAT_UTF_32BE;
#else
static enum MTextFormat default_utf_16 = MTEXT_FORMAT_UTF_16LE;
static enum MTextFormat default_utf_32 = MTEXT_FORMAT_UTF_32LE;
#endif

/** Increment character position CHAR_POS and byte position BYTE_POS
    so that they point to the next character in M-text MT.  No range
    check for CHAR_POS and BYTE_POS.  */

#define INC_POSITION(mt, char_pos, byte_pos)			\
  do {								\
    int c;							\
								\
    if ((mt)->format == MTEXT_FORMAT_UTF_8)			\
      {								\
	c = (mt)->data[(byte_pos)];				\
	(byte_pos) += CHAR_UNITS_BY_HEAD_UTF8 (c);		\
      }								\
    else							\
      {								\
	c = ((unsigned short *) ((mt)->data))[(byte_pos)];	\
								\
	if ((mt)->format != default_utf_16)			\
	  c = SWAP_16 (c);					\
	(byte_pos) += (c < 0xD800 || c >= 0xE000) ? 1 : 2;	\
      }								\
    (char_pos)++;						\
  } while (0)


/** Decrement character position CHAR_POS and byte position BYTE_POS
    so that they point to the previous character in M-text MT.  No
    range check for CHAR_POS and BYTE_POS.  */

#define DEC_POSITION(mt, char_pos, byte_pos)				\
  do {									\
    if ((mt)->format == MTEXT_FORMAT_UTF_8)				\
      {									\
	unsigned char *p1 = (mt)->data + (byte_pos);			\
	unsigned char *p0 = p1 - 1;					\
									\
	while (! CHAR_HEAD_P (p0)) p0--;				\
	(byte_pos) -= (p1 - p0);					\
      }									\
    else								\
      {									\
	int c = ((unsigned short *) ((mt)->data))[(byte_pos) - 1];	\
									\
	if ((mt)->format != default_utf_16)				\
	  c = SWAP_16 (c);						\
	(byte_pos) -= (c < 0xD800 || c >= 0xE000) ? 1 : 2;		\
      }									\
    (char_pos)--;							\
  } while (0)


static int
compare (MText *mt1, int from1, int to1, MText *mt2, int from2, int to2)
{
  if (mt1->format == mt2->format
      && (mt1->format < MTEXT_FORMAT_UTF_8))
    {
      unsigned char *p1, *pend1, *p2, *pend2;

      p1 = mt1->data + mtext__char_to_byte (mt1, from1);
      pend1 = mt1->data + mtext__char_to_byte (mt1, to1);

      p2 = mt2->data + mtext__char_to_byte (mt2, from2);<commit_msg>(INC_POSITION): Handle format other than utf8 and utf16. (DEC_POSITION): Likewise.<commit_after>/* mtext.c -- M-text module.
   Copyright (C) 2003, 2004
     National Institute of Advanced Industrial Science and Technology (AIST)
     Registration Number H15PRO112

   This file is part of the m17n library.

   The m17n library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public License
   as published by the Free Software Foundation; either version 2.1 of
   the License, or (at your option) any later version.

   The m17n library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the m17n library; if not, write to the Free
   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   02111-1307, USA.  */

/***en
    @addtogroup m17nMtext
    @brief M-text objects and API for them.

    In the m17n library, text is represented as an object called @e
    M-text rather than as a C-string (<tt>char *</tt> or <tt>unsigned
    char *</tt>).  An M-text is a sequence of characters whose length
    is equals to or more than 0, and can be coined from various
    character sources, e.g. C-strings, files, character codes, etc.

    M-texts are more useful than C-strings in the following points.

    @li M-texts can handle mixture of characters of various scripts,
    including all Unicode characters and more.  This is an
    indispensable facility when handling multilingual text.

    @li Each character in an M-text can have properties called @e text
    @e properties. Text properties store various kinds of information
    attached to parts of an M-text to provide application programs
    with a unified view of those information.  As rich information can
    be stored in M-texts in the form of text properties, functions in
    application programs can be simple.

    In addition, the library provides many functions to manipulate an
    M-text just the same way as a C-string.  */

/***ja
    @addtogroup m17nMtext

    @brief M-text ֥ȤȤ˴ؤ API.

    m17n 饤֥ϡ C-string<tt>char *</tt>  <tt>unsigned
    char *</tt>ˤǤϤʤ @e M-text ȸƤ֥֥ȤǥƥȤɽ롣
    M-text Ĺ 0 ʾʸǤꡢʸʤȤ 
    C-stringե롢ʸˤǤ롣

    M-text ˤϡC-string ˤʤʲħ롣     

    @li M-text ¿μʸƱˡߤơƱ
    ȤǤ롣Unicode ƤʸϤ󡢤¿ʸ
    ǰ롣¿ƥȤ򰷤ǤɬܤεǽǤ롣

    @li M-text γʸϡ@e ƥȥץѥƥ ȸƤФץѥƥ
    ĤȤǤ롣ƥȥץѥƥˤäơƥȤγ̤
    ؤ͡ʾ M-text ݻ뤳ȤǤ롣Τᡢ
    ξ򥢥ץꥱץŪ˰ȤǤ롣
    ޤM-text Τ˭٤ʾĤᡢץꥱץ
    γƴؿǲ뤳ȤǤ롣

    m17n 饤֥ϡ C-string 뤿󶡤
    δؿƱΤΤ M-text 뤿˥ݡȤƤ롣  */

/*=*/

#if !defined (FOR_DOXYGEN) || defined (DOXYGEN_INTERNAL_MODULE)
/*** @addtogroup m17nInternal
     @{ */

#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <locale.h>

#include "m17n.h"
#include "m17n-misc.h"
#include "internal.h"
#include "textprop.h"
#include "character.h"
#include "mtext.h"
#include "plist.h"

static M17NObjectArray mtext_table;

static MSymbol M_charbag;

#ifdef WORDS_BIGENDIAN
static enum MTextFormat default_utf_16 = MTEXT_FORMAT_UTF_16BE;
static enum MTextFormat default_utf_32 = MTEXT_FORMAT_UTF_32BE;
#else
static enum MTextFormat default_utf_16 = MTEXT_FORMAT_UTF_16LE;
static enum MTextFormat default_utf_32 = MTEXT_FORMAT_UTF_32LE;
#endif

/** Increment character position CHAR_POS and byte position UNIT_POS
    so that they point to the next character in M-text MT.  No range
    check for CHAR_POS and UNIT_POS.  */

#define INC_POSITION(mt, char_pos, unit_pos)			\
  do {								\
    int c;							\
								\
    if ((mt)->format <= MTEXT_FORMAT_UTF_8)			\
      {								\
	c = (mt)->data[(unit_pos)];				\
	(unit_pos) += CHAR_UNITS_BY_HEAD_UTF8 (c);		\
      }								\
    else if ((mt)->format <= MTEXT_FORMAT_UTF_16BE)		\
      {								\
	c = ((unsigned short *) ((mt)->data))[(unit_pos)];	\
								\
	if ((mt)->format != default_utf_16)			\
	  c = SWAP_16 (c);					\
	(unit_pos) += (c < 0xD800 || c >= 0xE000) ? 1 : 2;	\
      }								\
    else							\
      (unit_pos)++;						\
    (char_pos)++;						\
  } while (0)


/** Decrement character position CHAR_POS and byte position UNIT_POS
    so that they point to the previous character in M-text MT.  No
    range check for CHAR_POS and UNIT_POS.  */

#define DEC_POSITION(mt, char_pos, unit_pos)				\
  do {									\
    if ((mt)->format <= MTEXT_FORMAT_UTF_8)				\
      {									\
	unsigned char *p1 = (mt)->data + (unit_pos);			\
	unsigned char *p0 = p1 - 1;					\
									\
	while (! CHAR_HEAD_P (p0)) p0--;				\
	(unit_pos) -= (p1 - p0);					\
      }									\
    else if ((mt)->format <= MTEXT_FORMAT_UTF_16BE)			\
      {									\
	int c = ((unsigned short *) ((mt)->data))[(unit_pos) - 1];	\
									\
	if ((mt)->format != default_utf_16)				\
	  c = SWAP_16 (c);						\
	(unit_pos) -= (c < 0xD800 || c >= 0xE000) ? 1 : 2;		\
      }									\
    else								\
      (unit_pos)--;							\
    (char_pos)--;							\
  } while (0)


static int
compare (MText *mt1, int from1, int to1, MText *mt2, int from2, int to2)
{
  if (mt1->format == mt2->format
      && (mt1->format < MTEXT_FORMAT_UTF_8))
    {
      unsigned char *p1, *pend1, *p2, *pend2;

      p1 = mt1->data + mtext__char_to_byte (mt1, from1);
      pend1 = mt1->data + mtext__char_to_byte (mt1, to1);

      p2 = mt2->data + mtext__char_to_byte (mt2, from2);
{'fraction_non_alphanumeric': 0.10430754979157017, 'fraction_numerical': 0.026956924502084298, 'mean_word_length': 2.608288770053476, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7406716', 'n_tokens_mistral': 4739, 'n_tokens_neox': 4227, 'n_words': 1242}