<commit_before>#define PCRE2_CODE_UNIT_WIDTH 8

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/types.h>
#include <pcre2.h>

char *read_file(char *filename)
{
  char *data;
  long length = 0;

  FILE *fh = fopen(filename, "rb");

  fseek(fh, 0L, SEEK_END);
  length = ftell(fh);
  rewind(fh);
  data = malloc(length);

  fread(data, length, 1, fh);
  fclose(fh);

  return data;
}

void measure(char *data, char *pattern)
{
  int count = 0;
  double elapsed;
  struct timespec start, end;

  pcre2_code *re;
  int rc;
  int length = 0;
  int errornumber;
  PCRE2_SIZE erroroffset;
  PCRE2_SIZE offset = 0;
  pcre2_match_data *match_data;
  PCRE2_SPTR subject;
  size_t subject_length;
  PCRE2_SIZE *ovector;

  clock_gettime(CLOCK_MONOTONIC, &start);

  length = strlen(data);
  re = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, NULL);
  match_data = pcre2_match_data_create_from_pattern(re, NULL);

  while (pcre2_match(re, (PCRE2_SPTR8)data, strlen(data), offset, PCRE2_NOTBOL | PCRE2_NOTEMPTY, match_data, NULL) == 1)
  {
    count++;

    ovector = pcre2_get_ovector_pointer(match_data);
    offset = ovector[1];
  }

  clock_gettime(CLOCK_MONOTONIC, &end);
  elapsed = ((end.tv_sec - start.tv_sec) * 1E+9 + end.tv_nsec - start.tv_nsec) / 1E+6;

  printf("%f - %d\n", elapsed, count);
}

int main(int argc, char **argv)
{
  if (argc != 2)
  {
    printf("Usage: benchmark  <filename>");
    exit(1);
  }

  char *data = read_file(argv[1]);

  // Email
  measure(data, "[\\w\\.+-]+@[\\w\\.-]+\\.[\\w\\.-]+");

  // URI
  measure(data, "[\\w]+://[^/\\s?#]+[^\\s?#]+(?:\\?[^\\s#]*)?(?:#[^\\s]*)?");

  // IP
  measure(data, "(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])");

  free(data);

  return 0;
}
<commit_msg>Improve C PCRE benchmark. Use JIT and avoid data length recalculation.<commit_after>#include <stdio.h>
#include <string.h>
#include <sys/time.h>

#define PCRE2_CODE_UNIT_WIDTH 8
#include <pcre2.h>

char *read_file(char *filename)
{
  char *data;
  long length = 0;

  FILE *fh = fopen(filename, "rb");

  fseek(fh, 0, SEEK_END);
  length = ftell(fh);
  fseek(fh, 0, SEEK_SET);

  data = malloc(length);

  fread(data, length, 1, fh);
  fclose(fh);

  return data;
}

void measure(char *data, char *pattern)
{
  int count = 0;
  double elapsed;
  struct timespec start, end;
  pcre2_code *re;
  int errorcode;
  PCRE2_SIZE erroroffset;
  pcre2_match_context *match_ctx;
  pcre2_jit_stack *stack;
  pcre2_match_data *match_data;
  int length;
  PCRE2_SIZE offset = 0;
  PCRE2_SIZE *ovector;

  clock_gettime(CLOCK_MONOTONIC, &start);

  re = pcre2_compile((PCRE2_SPTR) pattern, PCRE2_ZERO_TERMINATED, PCRE2_UTF, &errorcode, &erroroffset, NULL);

  match_ctx = pcre2_match_context_create(NULL);
  pcre2_jit_compile(re, PCRE2_JIT_COMPLETE);
  stack = pcre2_jit_stack_create(0, 16384, NULL);
  pcre2_jit_stack_assign(match_ctx, NULL, stack);

  match_data = pcre2_match_data_create(1, NULL);
  length = strlen(data);

  while (pcre2_jit_match(re, (PCRE2_SPTR8) data, length, offset, 0, match_data, match_ctx) == 1)
  {
    count++;

    ovector = pcre2_get_ovector_pointer(match_data);
    offset = ovector[1];
  }

  clock_gettime(CLOCK_MONOTONIC, &end);
  elapsed = ((end.tv_sec - start.tv_sec) * 1e9 + end.tv_nsec - start.tv_nsec) / 1e6;

  printf("%f - %d\n", elapsed, count);

  pcre2_jit_stack_free(stack);
  pcre2_match_context_free(match_ctx);
  pcre2_match_data_free(match_data);
  pcre2_code_free(re);
}

int main(int argc, char **argv)
{
  if (argc != 2)
  {
    printf("Usage: benchmark  <filename>");
    exit(1);
  }

  char *data = read_file(argv[1]);

  // Email
  measure(data, "[\\w\\.+-]+@[\\w\\.-]+\\.[\\w\\.-]+");

  // URI
  measure(data, "[\\w]+://[^/\\s?#]+[^\\s?#]+(?:\\?[^\\s#]*)?(?:#[^\\s]*)?");

  // IP
  measure(data, "(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])");

  free(data);

  return 0;
}

{'fraction_non_alphanumeric': 0.19990042320139406, 'fraction_numerical': 0.037341299477221805, 'mean_word_length': 3.9482758620689653, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5164487', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1769, 'n_words': 315}