<commit_before>/*	$FreeBSD$	*/

/*-
 * Copyright (c) 1999 <NAME> and <NAME>
 * Copyright (C) 2008-2011 <NAME> <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "glue.h"

#include <ctype.h>
#include <limits.h>
#include <regex.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#ifdef TRE_WCHAR
#include <wchar.h>
#include <wctype.h>
#endif

#include "hashtable.h"
#include "tre-fastmatch.h"
#include "xmalloc.h"

static int	fastcmp(const fastmatch_t *fg, const void *data,
			tre_str_type_t type);

/*
 * Clean up if pattern compilation fails.
 */
#define FAIL_COMP(errcode)						\
  {									\
    if (fg->pattern)							\
      xfree(fg->pattern);						\
    if (fg->wpattern)							\
      xfree(fg->wpattern);						\
    if (fg->qsBc_table)							\
      hashtable_free(fg->qsBc_table);					\
    fg = NULL;								\
    return errcode;							\
  }

/*
 * Skips n characters in the input string and assigns the start
 * address to startptr. Note: as per IEEE Std 1003.1-2008
 * matching is based on bit pattern not character representations
 * so we can handle MB strings as byte sequences just like
 * SB strings.
 */
#define SKIP_CHARS(n)							\
  switch (type)								\
    {									\
      case STR_WIDE:							\
	startptr = str_wide + n;					\
	break;								\
      default:								\
	startptr = str_byte + n;					\
    }

/*
 * Converts the wide string pattern to SB/MB string and stores
 * it in fg->pattern. Sets fg->len to the byte length of the
 * converted string.
 */
#define STORE_MBS_PAT							\
  {									\
    size_t siz;								\
									\
    siz = wcstombs(NULL, fg->wpattern, 0);				\
    if (siz == (size_t)-1)						\
      return REG_BADPAT;						\
    fg->len = siz;							\
    fg->pattern = xmalloc(siz + 1);					\
    if (fg->pattern == NULL)						\
      return REG_ESPACE;						\
    wcstombs(fg->pattern, fg->wpattern, siz);				\
    fg->pattern[siz] = '\0';						\
  }									\

#define IS_OUT_OF_BOUNDS						\
  ((!fg->reversed							\
    ? ((type == STR_WIDE) ? ((j + fg->wlen) > len)			\
			  : ((j + fg->len) > len))			\
    : (j < 0)))

/*
 * Checks whether the new position after shifting in the input string
 * is out of the bounds and break out from the loop if so.
 */
#define CHECKBOUNDS							\
  if (IS_OUT_OF_BOUNDS)							\
    break;								\

/*
 * Shifts in the input string after a mismatch. The position of the
 * mismatch is stored in the mismatch variable.
 */
#define SHIFT								\
  CHECKBOUNDS;								\
									\
  {									\
    int r = -1;								\
    unsigned int bc = 0, gs = 0, ts;					\
									\
    switch (type)							\
      {									\
	case STR_WIDE:							\
	  if (!fg->hasdot)						\
	    {								\
	      if (u != 0 && (unsigned)mismatch == fg->wlen - 1 - shift)	\
		mismatch -= u;						\
	      v = fg->wlen - 1 - mismatch;				\
	      r = hashtable_get(fg->qsBc_table,				\
		&str_wide[!fg->reversed ? (size_t)j + fg->wlen		\
			  : (size_t)j - 1], &bc);			\
	      gs = fg->bmGs[mismatch];					\
	    }								\
	    bc = (r == HASH_OK) ? bc : fg->defBc;			\
	    DPRINT(("tre_fast_match: mismatch on character" CHF ", "	\
		    "BC %d, GS %d\n",					\
		    ((const tre_char_t *)startptr)[mismatch + 1],	\
		    bc, gs));						\
            break;							\
	default:							\
	  if (!fg->hasdot)						\
	    {								\
	      if (u != 0 && (unsigned)mismatch == fg->len - 1 - shift)	\
		mismatch -= u;						\
	      v = fg->len - 1 - mismatch;				\
	      gs = fg->sbmGs[mismatch];					\
	    }								\
	  bc = fg->qsBc[((const unsigned char *)str_byte)		\
			[!fg->reversed ? (size_t)j + fg->len		\
			 : (size_t)j - 1]];				\
	  DPRINT(("tre_fast_match: mismatch on character %c, "		\
		 "BC %d, GS %d\n",					\
		 ((const unsigned char *)startptr)[mismatch + 1],	\
		 bc, gs));						\
      }									\
    if (fg->hasdot)							\
      shift = bc;							\
    else								\
      {									\
	ts = ((long)u - v < 0) ? 0 : (u - v);				\
	shift = MAX(ts, bc);						\
	shift = MAX(shift, gs);						\
	if (shift == gs)						\
	  u = MIN((type == STR_WIDE ? fg->wlen : fg->len) - shift, v);	\
	else								\
	  {								\
	    if (ts < bc)						\
	      shift = MAX(shift, u + 1);				\
	    u = 0;							\
	  }								\
      }									\
      DPRINT(("tre_fast_match: shifting %u characters\n", shift));	\
      j = !fg->reversed ? j + shift : j - shift;			\
  }
<commit_msg>Fix build on i386 and arm.<commit_after>/*	$FreeBSD$	*/

/*-
 * Copyright (c) 1999 <NAME> and <NAME>
 * Copyright (C) 2008-2011 <NAME> <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "glue.h"

#include <ctype.h>
#include <limits.h>
#include <regex.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#ifdef TRE_WCHAR
#include <wchar.h>
#include <wctype.h>
#endif

#include "hashtable.h"
#include "tre-fastmatch.h"
#include "xmalloc.h"

static int	fastcmp(const fastmatch_t *fg, const void *data,
			tre_str_type_t type);

/*
 * Clean up if pattern compilation fails.
 */
#define FAIL_COMP(errcode)						\
  {									\
    if (fg->pattern)							\
      xfree(fg->pattern);						\
    if (fg->wpattern)							\
      xfree(fg->wpattern);						\
    if (fg->qsBc_table)							\
      hashtable_free(fg->qsBc_table);					\
    fg = NULL;								\
    return errcode;							\
  }

/*
 * Skips n characters in the input string and assigns the start
 * address to startptr. Note: as per IEEE Std 1003.1-2008
 * matching is based on bit pattern not character representations
 * so we can handle MB strings as byte sequences just like
 * SB strings.
 */
#define SKIP_CHARS(n)							\
  switch (type)								\
    {									\
      case STR_WIDE:							\
	startptr = str_wide + n;					\
	break;								\
      default:								\
	startptr = str_byte + n;					\
    }

/*
 * Converts the wide string pattern to SB/MB string and stores
 * it in fg->pattern. Sets fg->len to the byte length of the
 * converted string.
 */
#define STORE_MBS_PAT							\
  {									\
    size_t siz;								\
									\
    siz = wcstombs(NULL, fg->wpattern, 0);				\
    if (siz == (size_t)-1)						\
      return REG_BADPAT;						\
    fg->len = siz;							\
    fg->pattern = xmalloc(siz + 1);					\
    if (fg->pattern == NULL)						\
      return REG_ESPACE;						\
    wcstombs(fg->pattern, fg->wpattern, siz);				\
    fg->pattern[siz] = '\0';						\
  }									\

#define IS_OUT_OF_BOUNDS						\
  ((!fg->reversed							\
    ? ((type == STR_WIDE) ? ((j + fg->wlen) > len)			\
			  : ((j + fg->len) > len))			\
    : (j < 0)))

/*
 * Checks whether the new position after shifting in the input string
 * is out of the bounds and break out from the loop if so.
 */
#define CHECKBOUNDS							\
  if (IS_OUT_OF_BOUNDS)							\
    break;								\

/*
 * Shifts in the input string after a mismatch. The position of the
 * mismatch is stored in the mismatch variable.
 */
#define SHIFT								\
  CHECKBOUNDS;								\
									\
  {									\
    int r = -1;								\
    unsigned int bc = 0, gs = 0, ts;					\
									\
    switch (type)							\
      {									\
	case STR_WIDE:							\
	  if (!fg->hasdot)						\
	    {								\
	      if (u != 0 && (unsigned)mismatch == fg->wlen - 1 - shift)	\
		mismatch -= u;						\
	      v = fg->wlen - 1 - mismatch;				\
	      r = hashtable_get(fg->qsBc_table,				\
		&str_wide[!fg->reversed ? (size_t)j + fg->wlen		\
			  : (size_t)j - 1], &bc);			\
	      gs = fg->bmGs[mismatch];					\
	    }								\
	    bc = (r == HASH_OK) ? bc : fg->defBc;			\
	    DPRINT(("tre_fast_match: mismatch on character" CHF ", "	\
		    "BC %d, GS %d\n",					\
		    ((const tre_char_t *)startptr)[mismatch + 1],	\
		    bc, gs));						\
            break;							\
	default:							\
	  if (!fg->hasdot)						\
	    {								\
	      if (u != 0 && (unsigned)mismatch == fg->len - 1 - shift)	\
		mismatch -= u;						\
	      v = fg->len - 1 - mismatch;				\
	      gs = fg->sbmGs[mismatch];					\
	    }								\
	  bc = fg->qsBc[((const unsigned char *)str_byte)		\
			[!fg->reversed ? (size_t)j + fg->len		\
			 : (size_t)j - 1]];				\
	  DPRINT(("tre_fast_match: mismatch on character %c, "		\
		 "BC %d, GS %d\n",					\
		 ((const unsigned char *)startptr)[mismatch + 1],	\
		 bc, gs));						\
      }									\
    if (fg->hasdot)							\
      shift = bc;							\
    else								\
      {									\
	ts = (u >= v) ? (u - v) : 0;					\
	shift = MAX(ts, bc);						\
	shift = MAX(shift, gs);						\
	if (shift == gs)						\
	  u = MIN((type == STR_WIDE ? fg->wlen : fg->len) - shift, v);	\
	else								\
	  {								\
	    if (ts < bc)						\
	      shift = MAX(shift, u + 1);				\
	    u = 0;							\
	  }								\
      }									\
      DPRINT(("tre_fast_match: shifting %u characters\n", shift));	\
      j = !fg->reversed ? j + shift : j - shift;			\
  }

{'fraction_non_alphanumeric': 0.1299744110120886, 'fraction_numerical': 0.008117885820171181, 'mean_word_length': 1.8455937735375345, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 108, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1668212', 'n_tokens_mistral': 5154, 'n_tokens_neox': 4035, 'n_words': 1166}