<commit_before>/**
 * @file grok.h
 */
#ifndef _GROK_H_
#define _GROK_H_

#include <pcre.h>
#include <tcutil.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>

/*
 * A regular expression super library.
 */

typedef struct grok grok_t;

typedef struct grok_pattern {
  const char *name;
  char *regexp;
} grok_pattern_t;

struct grok {
  /** The original pattern given to grok_compile() */
  const char *pattern;

  /** length of the pattern string */
  int pattern_len;

  /** The full expanded pattern generated from grok_compile() */
  char *full_pattern;

  /** full_pattern string length */
  int full_pattern_len;

  /** tokyocabinet TCTREE of patterns */
  TCTREE *patterns;

  pcre *re;
  int *pcre_capture_vector;
  int pcre_num_captures;
  
  /* Data storage for named-capture (grok capture) information */
  TCTREE *captures_by_id;
  TCTREE *captures_by_name;
  TCTREE *captures_by_subname;
  TCTREE *captures_by_capture_number;
  int max_capture_num;
  
  /** PCRE pattern compilation errors */
  const char *pcre_errptr;
  int pcre_erroffset;
  int pcre_errno;

  unsigned int logmask;
  unsigned int logdepth;
  char *errstr;
};

extern int g_grok_global_initialized;
extern pcre *g_pattern_re;
extern int g_pattern_num_captures;
extern int g_cap_name;
extern int g_cap_pattern;
extern int g_cap_subname;
extern int g_cap_predicate;

/* pattern to match %{FOO:BAR} */
/* or %{FOO<=3} */

#define PATTERN_REGEX \
  "(?!<\\\\)%{" \
  "(?<name>" \
    "(?<pattern>[A-z0-9]+)" \
    "(?::(?<subname>[A-z0-9]+))?" \
  ")" \
  "\\s*(?<predicate>" \
    "(?:" \
      "(?P<curly>\\{(?:(?>[^{}]+|(?>\\\\[{}])+)|(?P>curly))*\\})" \
      "|" \
      "(?:[^{}]+|\\\\[{}])+" \
    ")+" \
  ")?" \
  "}"

/** Safe return code */
#define GROK_OK 0

/** File not accessible. This occurs when you try to load patterns from
 * a file and grok cannot read it. */
#define GROK_ERROR_FILE_NOT_ACCESSIBLE 1

/** Pattern not found is given if you try to search for a pattern
 * by name that is not known, that is, it wasn't added with
 * grok_pattern_add or grok_patterns_import* */
#define GROK_ERROR_PATTERN_NOT_FOUND 2

#define GROK_ERROR_UNEXPECTED_READ_SIZE 3<commit_msg>- Allow '_' and ':' in grok pattern subname.   This should help solve: http://code.google.com/p/logstash/issues/detail?id=45<commit_after>/**
 * @file grok.h
 */
#ifndef _GROK_H_
#define _GROK_H_

#include <pcre.h>
#include <tcutil.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>

/*
 * A regular expression super library.
 */

typedef struct grok grok_t;

typedef struct grok_pattern {
  const char *name;
  char *regexp;
} grok_pattern_t;

struct grok {
  /** The original pattern given to grok_compile() */
  const char *pattern;

  /** length of the pattern string */
  int pattern_len;

  /** The full expanded pattern generated from grok_compile() */
  char *full_pattern;

  /** full_pattern string length */
  int full_pattern_len;

  /** tokyocabinet TCTREE of patterns */
  TCTREE *patterns;

  pcre *re;
  int *pcre_capture_vector;
  int pcre_num_captures;
  
  /* Data storage for named-capture (grok capture) information */
  TCTREE *captures_by_id;
  TCTREE *captures_by_name;
  TCTREE *captures_by_subname;
  TCTREE *captures_by_capture_number;
  int max_capture_num;
  
  /** PCRE pattern compilation errors */
  const char *pcre_errptr;
  int pcre_erroffset;
  int pcre_errno;

  unsigned int logmask;
  unsigned int logdepth;
  char *errstr;
};

extern int g_grok_global_initialized;
extern pcre *g_pattern_re;
extern int g_pattern_num_captures;
extern int g_cap_name;
extern int g_cap_pattern;
extern int g_cap_subname;
extern int g_cap_predicate;

/* pattern to match %{FOO:BAR} */
/* or %{FOO<=3} */

#define PATTERN_REGEX \
  "(?!<\\\\)%{" \
  "(?<name>" \
    "(?<pattern>[A-z0-9]+)" \
    "(?::(?<subname>[A-z0-9_:]+))?" \
  ")" \
  "\\s*(?<predicate>" \
    "(?:" \
      "(?P<curly>\\{(?:(?>[^{}]+|(?>\\\\[{}])+)|(?P>curly))*\\})" \
      "|" \
      "(?:[^{}]+|\\\\[{}])+" \
    ")+" \
  ")?" \
  "}"

/** Safe return code */
#define GROK_OK 0

/** File not accessible. This occurs when you try to load patterns from
 * a file and grok cannot read it. */
#define GROK_ERROR_FILE_NOT_ACCESSIBLE 1

/** Pattern not found is given if you try to search for a pattern
 * by name that is not known, that is, it wasn't added with
 * grok_pattern_add or grok_patterns_import* */
#define GROK_ERROR_PATTERN_NOT_FOUND 2

#define GROK_ERROR_UNEXPECTED_READ_SIZE 3
{'fraction_non_alphanumeric': 0.15541288465795883, 'fraction_numerical': 0.004427717511622758, 'mean_word_length': 3.6196319018404908, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6279143', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1694, 'n_words': 438}