<commit_before>/*-------------------------------------------------------------------------
 *
 * unaccent.c
 *    Text search unaccent dictionary
 *
 * Copyright (c) 2009, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *    $PostgreSQL: pgsql/contrib/unaccent/unaccent.c,v 1.2 2009/08/18 15:37:04 tgl Exp $
 *
 *-------------------------------------------------------------------------
 */

#include "postgres.h"

#include "fmgr.h"
#include "catalog/namespace.h"
#include "commands/defrem.h"
#include "mb/pg_wchar.h"
#include "tsearch/ts_cache.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_public.h"
#include "utils/builtins.h"

PG_MODULE_MAGIC;

/*
 * Unaccent dictionary uses uncompressed suffix tree to find a 
 * character to replace. Each node of tree is an array of 
 * SuffixChar struct with length = 256 (n-th element of array
 * corresponds to byte)
 */
typedef struct SuffixChar {
	struct SuffixChar	*nextChar;
	char				*replaceTo;
	int					replacelen;
} SuffixChar;

/*
 * placeChar - put str into tree's structure, byte by byte.
 */
static SuffixChar*
placeChar(SuffixChar *node, unsigned char *str, int lenstr, char *replaceTo, int replacelen)
{
	SuffixChar	*curnode;

	if ( !node )
	{
		node = palloc(sizeof(SuffixChar) * 256);
		memset(node, 0, sizeof(SuffixChar) * 256);
	}

	curnode = node + *str;

	if ( lenstr == 1 )
	{
		if ( curnode->replaceTo )
			elog(WARNING, "duplicate TO argument, use first one");
		else
		{
			curnode->replacelen = replacelen;
			curnode->replaceTo = palloc( replacelen );
			memcpy(curnode->replaceTo, replaceTo, replacelen);
		}
	}
	else
	{
		curnode->nextChar = placeChar( curnode->nextChar, str+1, lenstr-1, replaceTo, replacelen);
	}

	return node;
}

/*
 * initSuffixTree  - create suffix tree from file. Function converts
 * UTF8-encoded file into current encoding.
 */
static SuffixChar*
initSuffixTree(char *filename) 
{
	SuffixChar *rootSuffixTree = NULL;
	MemoryContext ccxt = CurrentMemoryContext;
	tsearch_readline_state	trst;
	bool			skip;

	filename = get_tsearch_config_filename(filename, "rules");
	if (!tsearch_readline_begin(&trst, filename))
		ereport(ERROR,
				(errcode(ERRCODE_CONFIG_FILE_ERROR),
				 errmsg("could not open unaccent file \"%s\": %m",
						filename)));

	do	
	{
		char	src[4096];
		char	trg[4096];
		int		srclen;
		int		trglen;
		char   *line = NULL;

		skip = true;

		PG_TRY();
		{
			/*
			 * pg_do_encoding_conversion() (called by tsearch_readline())
			 * will emit exception if it finds untranslatable characters in current locale.
			 * We just skip such characters.<commit_msg>Fix some *other* compiler warnings from a different gcc version.<commit_after>/*-------------------------------------------------------------------------
 *
 * unaccent.c
 *    Text search unaccent dictionary
 *
 * Copyright (c) 2009, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *    $PostgreSQL: pgsql/contrib/unaccent/unaccent.c,v 1.3 2009/08/18 15:57:26 tgl Exp $
 *
 *-------------------------------------------------------------------------
 */

#include "postgres.h"

#include "fmgr.h"
#include "catalog/namespace.h"
#include "commands/defrem.h"
#include "mb/pg_wchar.h"
#include "tsearch/ts_cache.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_public.h"
#include "utils/builtins.h"

PG_MODULE_MAGIC;

/*
 * Unaccent dictionary uses uncompressed suffix tree to find a 
 * character to replace. Each node of tree is an array of 
 * SuffixChar struct with length = 256 (n-th element of array
 * corresponds to byte)
 */
typedef struct SuffixChar {
	struct SuffixChar	*nextChar;
	char				*replaceTo;
	int					replacelen;
} SuffixChar;

/*
 * placeChar - put str into tree's structure, byte by byte.
 */
static SuffixChar*
placeChar(SuffixChar *node, unsigned char *str, int lenstr, char *replaceTo, int replacelen)
{
	SuffixChar	*curnode;

	if ( !node )
	{
		node = palloc(sizeof(SuffixChar) * 256);
		memset(node, 0, sizeof(SuffixChar) * 256);
	}

	curnode = node + *str;

	if ( lenstr == 1 )
	{
		if ( curnode->replaceTo )
			elog(WARNING, "duplicate TO argument, use first one");
		else
		{
			curnode->replacelen = replacelen;
			curnode->replaceTo = palloc( replacelen );
			memcpy(curnode->replaceTo, replaceTo, replacelen);
		}
	}
	else
	{
		curnode->nextChar = placeChar( curnode->nextChar, str+1, lenstr-1, replaceTo, replacelen);
	}

	return node;
}

/*
 * initSuffixTree  - create suffix tree from file. Function converts
 * UTF8-encoded file into current encoding.
 */
static SuffixChar*
initSuffixTree(char *filename) 
{
	SuffixChar * volatile rootSuffixTree = NULL;
	MemoryContext ccxt = CurrentMemoryContext;
	tsearch_readline_state	trst;
	volatile bool	skip;

	filename = get_tsearch_config_filename(filename, "rules");
	if (!tsearch_readline_begin(&trst, filename))
		ereport(ERROR,
				(errcode(ERRCODE_CONFIG_FILE_ERROR),
				 errmsg("could not open unaccent file \"%s\": %m",
						filename)));

	do	
	{
		char	src[4096];
		char	trg[4096];
		int		srclen;
		int		trglen;
		char   *line = NULL;

		skip = true;

		PG_TRY();
		{
			/*
			 * pg_do_encoding_conversion() (called by tsearch_readline())
			 * will emit exception if it finds untranslatable characters in current locale.
			 * We just skip such characters.
{'fraction_non_alphanumeric': 0.16546077684691546, 'fraction_numerical': 0.015993907083015995, 'mean_word_length': 3.0099236641221374, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7120332', 'n_tokens_mistral': 1969, 'n_tokens_neox': 1769, 'n_words': 466}