<commit_before>/*************************************************************************/
/* Copyright (c) 2005  <NAME>                                     */
/* Copyright (c) 2009 <NAME>                                      */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

/* On MS Windows, regex.h fails to pull in size_t, so work around this by
 * including <stddef.h> before <regex.h> (<sys/types.h> is not enough) */
#include <stddef.h>
#if HAVE_TRE_TRE_H
#include <tre/regex.h>
#else
#include <regex.h>
#endif
#include "api-structures.h"
#include "error.h"          /* verbosity */
#include "externs.h"        /* lgdebug() */
#include "dict-common/dict-api.h"
#include "dict-common/dict-common.h"
#include "dict-common/regex-morph.h"
#include "link-includes.h"


/**
 * Support for the regular-expression based token matching system
 * using standard POSIX regex.
 */

/**
 * Notify an error message according to the error code.
 */
static void prt_regerror(const char *msg, const Regex_node *re, int rc)
{
	const size_t errbuf_size = regerror(rc, re->re, NULL, 0);
	char * const errbuf = malloc(errbuf_size);

	/*
	prt_error("Error: Failed to compile regex '%s' (%s) at %d: %s\n",
					re->pattern, re->name, erroroffset, error);
	*/
	regerror(rc, re->re, errbuf, errbuf_size);
	prt_error("Error: %s: \"%s\" (%s): %s\n", msg, re->pattern, re->name, errbuf);
	free(errbuf);
}

/**
 * Compiles all the given regexs. Returns 0 on success,
 * else an error code.
 */
int compile_regexs(Regex_node *re, Dictionary dict)
{
	regex_t *preg;
	int rc;

	while (re != NULL)
	{
		/* If re->re non-null, assume compiled already. */
		if(re->re == NULL)
		{
			/* Compile with default options (0) and default character
			 * tables (NULL). */
			/* re->re = pcre_compile(re->pattern, 0, &error, &erroroffset, NULL); */
			preg = (regex_t *) malloc (sizeof(regex_t));
			re->re = preg;

			/* REG_ENHANCED is needed for OS X to support \w etc. */
#ifndef REG_ENHANCED
#define REG_ENHANCED 0
#endif
			rc = regcomp(preg, re->pattern, REG_NOSUB|REG_EXTENDED|REG_ENHANCED);
			if (rc)
			{
				prt_regerror("Failed to compile regex", re, rc);
				return rc;
			}

			/* Check that the regex name is defined in the dictionary. */
			if ((NULL != dict) && !boolean_dictionary_lookup(dict, re->name))
			{
				/* TODO: better error handing. Maybe remove the regex? */
				prt_error("Error: Regex name %s not found in dictionary!\n",
				       re->name);
			}
		}
		re = re->next;
	}
	return 0;
}

/**
 * Tries to match each regex in turn to word s.
 * On match, returns the name of the first matching regex.
 * If no match is found, returns NULL.
 */
#define D_MRE 6
const char *match_regex(const Regex_node *re, const char *s)
{
	int rc;<commit_msg>Fix a possible crash due to pcre2-posix regcomp() failure<commit_after>/*************************************************************************/
/* Copyright (c) 2005  <NAME>                                     */
/* Copyright (c) 2009 <NAME>                                      */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

/* On MS Windows, regex.h fails to pull in size_t, so work around this by
 * including <stddef.h> before <regex.h> (<sys/types.h> is not enough) */
#include <stddef.h>
#if HAVE_TRE_TRE_H
#include <tre/regex.h>
#else
#include <regex.h>
#endif
#include "api-structures.h"
#include "error.h"          /* verbosity */
#include "externs.h"        /* lgdebug() */
#include "dict-common/dict-api.h"
#include "dict-common/dict-common.h"
#include "dict-common/regex-morph.h"
#include "link-includes.h"


/**
 * Support for the regular-expression based token matching system
 * using standard POSIX regex.
 */

/**
 * Notify an error message according to the error code.
 */
static void prt_regerror(const char *msg, const Regex_node *re, int rc)
{
	const size_t errbuf_size = regerror(rc, re->re, NULL, 0);
	char * const errbuf = malloc(errbuf_size);

	/*
	prt_error("Error: Failed to compile regex '%s' (%s) at %d: %s\n",
					re->pattern, re->name, erroroffset, error);
	*/
	regerror(rc, re->re, errbuf, errbuf_size);
	prt_error("Error: %s: \"%s\" (%s): %s\n", msg, re->pattern, re->name, errbuf);
	free(errbuf);
}

/**
 * Compiles all the given regexs. Returns 0 on success,
 * else an error code.
 */
int compile_regexs(Regex_node *re, Dictionary dict)
{
	regex_t *preg;
	int rc;

	while (re != NULL)
	{
		/* If re->re non-null, assume compiled already. */
		if(re->re == NULL)
		{
			/* Compile with default options (0) and default character
			 * tables (NULL). */
			/* re->re = pcre_compile(re->pattern, 0, &error, &erroroffset, NULL); */
			preg = (regex_t *) malloc (sizeof(regex_t));
			re->re = preg;

			/* REG_ENHANCED is needed for OS X to support \w etc. */
#ifndef REG_ENHANCED
#define REG_ENHANCED 0
#endif
			rc = regcomp(preg, re->pattern, REG_NOSUB|REG_EXTENDED|REG_ENHANCED);
			if (rc)
			{
				prt_regerror("Failed to compile regex", re, rc);
				re->re = NULL;
				return rc;
			}

			/* Check that the regex name is defined in the dictionary. */
			if ((NULL != dict) && !boolean_dictionary_lookup(dict, re->name))
			{
				/* TODO: better error handing. Maybe remove the regex? */
				prt_error("Error: Regex name %s not found in dictionary!\n",
				       re->name);
			}
		}
		re = re->next;
	}
	return 0;
}

/**
 * Tries to match each regex in turn to word s.
 * On match, returns the name of the first matching regex.
 * If no match is found, returns NULL.
 */
#define D_MRE 6
const char *match_regex(const Regex_node *re, const char *s)
{
	int rc;
{'fraction_non_alphanumeric': 0.17239309827456864, 'fraction_numerical': 0.004651162790697674, 'mean_word_length': 2.508421052631579, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '807505', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2014, 'n_words': 681}