<commit_before>/*----------------------------------------------------------------------------
 *
 * tokenizer.c
 *
 * Tokenization support functions
 *
 * Tokens are stored in a linked list to make manipulation easy.
 * We have support for three types of tokenization:
 * (i)   space: is treated as delimiter;
 * (ii)  non-alphanumeric: is treated as delimiter;
 * (iii) n-gram: token is a n-character "window".
 *
 *
 * Copyright (c) 2008-2012, <NAME>
 *
 *----------------------------------------------------------------------------
 */

#include "tokenizer.h"


TokenList *initTokenList(int a)
{
	TokenList	*t;

	t = (TokenList *) malloc(sizeof(TokenList));

	t->isset = a;
	t->size = 0;
	t->head = NULL;
	t->tail = NULL;

	elog(DEBUG4, "t->isset: %d", t->isset);

	return t;
}

void destroyTokenList(TokenList *t)
{
	char	*n;
	int		i;
	int		len;

	while (t->size > 0)
	{
		len = strlen(t->head->data);
		n = (char *) malloc(sizeof(char) * len + 1);
		strcpy(n, t->head->data);

		i = removeToken(t);
		if (i == 0)
			elog(DEBUG3, "token \"%s\" removed; actual token list size: %d", n, t->size);
		else
			elog(DEBUG3, "failed to remove token: \"%s\"", n);

		free(n);
	}

	free(t);
}

int addToken(TokenList *t, char *s)
{
	Token	*n;
	
	if (t->isset)
	{
		Token *x = searchToken(t, s);
		if (x != NULL)
		{
			x->freq++;

			elog(DEBUG3, "token \"%s\" is already in the list; frequency: %d", s, x->freq);

			return -1;
		}
	}

	n = (Token *) malloc(sizeof(Token));
	if (n == NULL)
		return -1;

	/*
	 * memory is allocated by tokenizeByXXX()
	 */
	n->data = s;
	n->freq = 1;	/* first token */

	if (t->size == 0)
		t->tail = n;

	n->next = t->head;
	t->head = n;

	t->size++;

	return 0;
}

/*
 * free up the head node and its content
 */
int removeToken(TokenList *t)
{
	Token	*n;

	if (t->size == 0)
	{
		elog(DEBUG3, "list is empty");

		return -1;
	}

	n = t->head;
	t->head = n->next;

	if (t->size == 1)
		t->tail = NULL;

	free(n->data);
	free(n);

	t->size--;

	return 0;
}

Token *searchToken(TokenList *t, char *s)
{
	Token	*n;

	n = t->head;
	while (n != NULL)
	{
#ifdef PGS_IGNORE_CASE
		/*
		 * For portability reason, use pg_strcasecmp instead of strcasecmp
		 * (Windows doesn't provide this function).
		 */
		if (pg_strcasecmp(n->data, s) == 0)
#else
		if (strcmp(n->data, s) == 0)
#endif
		{
			elog(DEBUG4, "\"%s\" found", n->data);

			return n;
		}
		n = n->next;
	}

	return NULL;
}

void printToken(TokenList *t)
{
	Token	*n;

	elog(DEBUG3, "===================================================");

	if (t->size == 0)
		elog(DEBUG3, "word list is empty");

	n = t->head;
	while (n != NULL)
	{
		elog(DEBUG3, "addr: %p; next: %p; word: %s; freq: %d", n, n->next, n->data, n->freq);

		n = n->next;
	}

	elog(DEBUG3, "head: %s", t->head->data);
	elog(DEBUG3, "tail: %s", t->tail->data);
	elog(DEBUG3, "===================================================");
}

/*
 * XXX non alnum characters are ignored in this function
 * XXX because they are treated as delimiter characters<commit_msg>Some debug messages doesn't check for NULL before echoing data. Reported by ericjredman.<commit_after>/*----------------------------------------------------------------------------
 *
 * tokenizer.c
 *
 * Tokenization support functions
 *
 * Tokens are stored in a linked list to make manipulation easy.
 * We have support for three types of tokenization:
 * (i)   space: is treated as delimiter;
 * (ii)  non-alphanumeric: is treated as delimiter;
 * (iii) n-gram: token is a n-character "window".
 *
 *
 * Copyright (c) 2008-2012, <NAME>
 *
 *----------------------------------------------------------------------------
 */

#include "tokenizer.h"


TokenList *initTokenList(int a)
{
	TokenList	*t;

	t = (TokenList *) malloc(sizeof(TokenList));

	t->isset = a;
	t->size = 0;
	t->head = NULL;
	t->tail = NULL;

	elog(DEBUG4, "t->isset: %d", t->isset);

	return t;
}

void destroyTokenList(TokenList *t)
{
	char	*n;
	int		i;
	int		len;

	while (t->size > 0)
	{
		len = strlen(t->head->data);
		n = (char *) malloc(sizeof(char) * len + 1);
		strcpy(n, t->head->data);

		i = removeToken(t);
		if (i == 0)
			elog(DEBUG3, "token \"%s\" removed; actual token list size: %d", n, t->size);
		else
			elog(DEBUG3, "failed to remove token: \"%s\"", n);

		free(n);
	}

	free(t);
}

int addToken(TokenList *t, char *s)
{
	Token	*n;
	
	if (t->isset)
	{
		Token *x = searchToken(t, s);
		if (x != NULL)
		{
			x->freq++;

			elog(DEBUG3, "token \"%s\" is already in the list; frequency: %d", s, x->freq);

			return -1;
		}
	}

	n = (Token *) malloc(sizeof(Token));
	if (n == NULL)
		return -1;

	/*
	 * memory is allocated by tokenizeByXXX()
	 */
	n->data = s;
	n->freq = 1;	/* first token */

	if (t->size == 0)
		t->tail = n;

	n->next = t->head;
	t->head = n;

	t->size++;

	return 0;
}

/*
 * free up the head node and its content
 */
int removeToken(TokenList *t)
{
	Token	*n;

	if (t->size == 0)
	{
		elog(DEBUG3, "list is empty");

		return -1;
	}

	n = t->head;
	t->head = n->next;

	if (t->size == 1)
		t->tail = NULL;

	free(n->data);
	free(n);

	t->size--;

	return 0;
}

Token *searchToken(TokenList *t, char *s)
{
	Token	*n;

	n = t->head;
	while (n != NULL)
	{
#ifdef PGS_IGNORE_CASE
		/*
		 * For portability reason, use pg_strcasecmp instead of strcasecmp
		 * (Windows doesn't provide this function).
		 */
		if (pg_strcasecmp(n->data, s) == 0)
#else
		if (strcmp(n->data, s) == 0)
#endif
		{
			elog(DEBUG4, "\"%s\" found", n->data);

			return n;
		}
		n = n->next;
	}

	return NULL;
}

void printToken(TokenList *t)
{
	Token	*n;

	elog(DEBUG3, "===================================================");

	if (t->size == 0)
		elog(DEBUG3, "word list is empty");

	n = t->head;
	while (n != NULL)
	{
		elog(DEBUG3, "addr: %p; next: %p; word: %s; freq: %d", n, n->next, n->data, n->freq);

		n = n->next;
	}

	if (t->head != NULL)
		elog(DEBUG3, "head: %s", t->head->data);
	if (t->tail != NULL)
		elog(DEBUG3, "tail: %s", t->tail->data);
	elog(DEBUG3, "===================================================");
}

/*
 * XXX non alnum characters are ignored in this function
 * XXX because they are treated as delimiter characters
{'fraction_non_alphanumeric': 0.2595606183889341, 'fraction_numerical': 0.011716842961757526, 'mean_word_length': 2.522063037249284, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541881', 'n_tokens_mistral': 2576, 'n_tokens_neox': 2352, 'n_words': 663}