<commit_before>/* FriBidi
 * fribidi-char-sets-cap-rtl.c - CapRTL character set conversion routines
 *
 * $Id: fribidi-char-sets-cap-rtl.c,v 1.9 2004-06-15 11:52:02 behdad Exp $
 * $Author: behdad $
 * $Date: 2004-06-15 11:52:02 $
 * $Revision: 1.9 $
 * $Source: /home/behdad/src/fdo/fribidi/togit/git/../fribidi/fribidi2/charset/fribidi-char-sets-cap-rtl.c,v $
 *
 * Authors:
 *   <NAME>, 2001, 2002, 2004
 *   <NAME>, 1999, 2000
 *
 * Copyright (C) 2004 Sharif FarsiWeb, Inc
 * Copyright (C) 2001,2002 <NAME>
 * Copyright (C) 1999,2000 <NAME>
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library, in a file named COPYING; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307, USA
 * 
 * For licensing issues, contact <<EMAIL>>.
 */

#include <common.h>

#include <fribidi-char-sets-cap-rtl.h>

#include <fribidi-unicode.h>
#include <fribidi-mirroring.h>
#include <fribidi-bidi-types.h>

#include <bidi-types.h>

#include <stdio.h>

enum MyFriBidiTypeEnum
{
# define _FRIBIDI_ADD_TYPE(TYPE,SYMBOL) TYPE = FRIBIDI_TYPE_##TYPE,
# include "fribidi-bidi-types-list.h"
# undef _FRIBIDI_ADD_TYPE
  _FRIBIDI_NUM_TYPES
};


static FriBidiCharType CapRTLCharTypes[] = {
/* *INDENT-OFF* */
  ON, ON, ON, ON, LTR,RTL,ON, ON, ON, ON, ON, ON, ON, BS, RLO,RLE, /* 00-0f */
  LRO,LRE,PDF,WS, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON,  /* 10-1f */
  WS, ON, ON, ON, ET, ON, ON, ON, ON, ON, ON, ET, CS, ON, ES, ES,  /* 20-2f */
  EN, EN, EN, EN, EN, EN, AN, AN, AN, AN, CS, ON, ON, ON, ON, ON,  /* 30-3f */
  RTL,AL, AL, AL, AL, AL, AL, RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL, /* 40-4f */
  RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,ON, BS, ON, BN, ON,  /* 50-5f */
  NSM,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR, /* 60-6f */
  LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,ON, SS, ON, WS, ON,  /* 70-7f */
/* *INDENT-ON* */
};

#define CAPRTL_CHARS (sizeof CapRTLCharTypes / sizeof CapRTLCharTypes[0])

static FriBidiChar *caprtl_to_unicode = NULL;

static void
init_cap_rtl (
  void
)
{
  int request[_FRIBIDI_NUM_TYPES];
  FriBidiCharType to_type[_FRIBIDI_NUM_TYPES];
  int num_types = 0, j, count = 0;
  FriBidiCharType i;

  caprtl_to_unicode =
    (FriBidiChar *) fribidi_malloc (CAPRTL_CHARS *
				    sizeof caprtl_to_unicode[0]);
  for (i = 0; i < CAPRTL_CHARS; i++)
    if (fribidi_get_mirror_char (i, NULL))
      caprtl_to_unicode[i] = i;
    else
      {
	for (j = 0; j < num_types; j++)
	  if (to_type[j] == CapRTLCharTypes[i])
	    break;
	if (j == num_types)
	  {
	    num_types++;
	    to_type[j] = CapRTLCharTypes[i];
	    request[j] = 0;
	  }
	request[j]++;
	count++;
      }
  for (i = 1; i < 0x10000 && count; i++)	/* Assign BMP chars to CapRTL entries */
    if (!fribidi_get_mirror_char (i, NULL))
      {
	int j, k;
	for (j = 0; j < num_types; j++)
	  if (to_type[j] == fribidi_get_bidi_type (i))
	    break;
	if (!request[j])	/* Do not need this type */
	  continue;
	for (k = 0; k < CAPRTL_CHARS; k++)
	  if (!caprtl_to_unicode[k] && to_type[j] == CapRTLCharTypes[k])
	    break;
	if (k < CAPRTL_CHARS)
	  {
	    request[j]--;
	    count--;
	    caprtl_to_unicode[k] = i;
	  }
      }
}

static char
fribidi_unicode_to_cap_rtl_c (<commit_msg>Better conversion table from CapRTL to Unicode.<commit_after>/* FriBidi
 * fribidi-char-sets-cap-rtl.c - CapRTL character set conversion routines
 *
 * $Id: fribidi-char-sets-cap-rtl.c,v 1.10 2005-01-10 06:43:53 behdad Exp $
 * $Author: behdad $
 * $Date: 2005-01-10 06:43:53 $
 * $Revision: 1.10 $
 * $Source: /home/behdad/src/fdo/fribidi/togit/git/../fribidi/fribidi2/charset/fribidi-char-sets-cap-rtl.c,v $
 *
 * Authors:
 *   <NAME>, 2001, 2002, 2004
 *   <NAME>, 1999, 2000
 *
 * Copyright (C) 2004 <NAME>, Inc
 * Copyright (C) 2001,2002 <NAME>
 * Copyright (C) 1999,2000 <NAME>
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library, in a file named COPYING; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307, USA
 * 
 * For licensing issues, contact <<EMAIL>>.
 */

#include <common.h>

#include <fribidi-char-sets-cap-rtl.h>

#include <fribidi-unicode.h>
#include <fribidi-mirroring.h>
#include <fribidi-bidi-types.h>

#include <bidi-types.h>

#include <stdio.h>

enum MyFriBidiTypeEnum
{
# define _FRIBIDI_ADD_TYPE(TYPE,SYMBOL) TYPE = FRIBIDI_TYPE_##TYPE,
# include "fribidi-bidi-types-list.h"
# undef _FRIBIDI_ADD_TYPE
  _FRIBIDI_NUM_TYPES
};


static FriBidiCharType CapRTLCharTypes[] = {
/* *INDENT-OFF* */
  ON, ON, ON, ON, LTR,RTL,ON, ON, ON, ON, ON, ON, ON, BS, RLO,RLE, /* 00-0f */
  LRO,LRE,PDF,WS, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON, ON,  /* 10-1f */
  WS, ON, ON, ON, ET, ON, ON, ON, ON, ON, ON, ET, CS, ON, ES, ES,  /* 20-2f */
  EN, EN, EN, EN, EN, EN, AN, AN, AN, AN, CS, ON, ON, ON, ON, ON,  /* 30-3f */
  RTL,AL, AL, AL, AL, AL, AL, RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL, /* 40-4f */
  RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,RTL,ON, BS, ON, BN, ON,  /* 50-5f */
  NSM,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR, /* 60-6f */
  LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,LTR,ON, SS, ON, WS, ON,  /* 70-7f */
/* *INDENT-ON* */
};

#define CAPRTL_CHARS (sizeof CapRTLCharTypes / sizeof CapRTLCharTypes[0])

static FriBidiChar *caprtl_to_unicode = NULL;

static void
init_cap_rtl (
  void
)
{
  int request[_FRIBIDI_NUM_TYPES];
  FriBidiCharType to_type[_FRIBIDI_NUM_TYPES];
  int num_types = 0, j, count = 0;
  FriBidiCharType i;
  char mark[CAPRTL_CHARS];

  caprtl_to_unicode =
    (FriBidiChar *) fribidi_malloc (CAPRTL_CHARS *
				    sizeof caprtl_to_unicode[0]);
  for (i = 0; i < CAPRTL_CHARS; i++)
    if (CapRTLCharTypes[i] == fribidi_get_bidi_type(i))
    {
      caprtl_to_unicode[i] = i;
      mark[i] = 1;
    }
    else
      {
	caprtl_to_unicode[i] = FRIBIDI_UNICODE_CHARS;
	mark[i] = 0;
        if (fribidi_get_mirror_char (i, NULL))
	  DBG ("warning: I could not map mirroring character map to itself in CapRTL");

	for (j = 0; j < num_types; j++)
	  if (to_type[j] == CapRTLCharTypes[i])
	    break;
	if (j == num_types)
	  {
	    num_types++;
	    to_type[j] = CapRTLCharTypes[i];
	    request[j] = 0;
	  }
	request[j]++;
	count++;
      }
  for (i = 0; i < 0x10000 && count; i++)	/* Assign BMP chars to CapRTL entries */
    if (!fribidi_get_mirror_char (i, NULL) && !(i < CAPRTL_CHARS && mark[i]))
      {
	int j, k;
	FriBidiCharType t = fribidi_get_bidi_type (i);
	for (j = 0; j < num_types; j++)
	  if (to_type[j] == t)
	    break;
	if (!request[j])	/* Do not need this type */
	  continue;
	for (k = 0; k < CAPRTL_CHARS; k++)
	  if (caprtl_to_unicode[k] == FRIBIDI_UNICODE_CHARS && to_type[j] == CapRTLCharTypes[k])
	  {
	    request[j]--;
	    count--;
	    caprtl_to_unicode[k] = i;
	    break;
	  }
      }
  if (count)
  {
    int j;

    DBG ("warning: could not find a mapping for CapRTL to Unicode:");
    for (j = 0; j < num_types; j++)
      if (request[j])
	DBG2 ("  need this type: %s", fribidi_get_bidi_type_name
	    (to_type[j]));
	
  }
}

static char
fribidi_unicode_to_cap_rtl_c (
{'fraction_non_alphanumeric': 0.15146341463414634, 'fraction_numerical': 0.03219512195121951, 'mean_word_length': 3.428185745140389, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2473207', 'n_tokens_mistral': 3756, 'n_tokens_neox': 3409, 'n_words': 885}