<commit_before>#ifndef RE_AST_H
#define RE_AST_H

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>

#include <re/re.h>

#include <fsm/fsm.h>
#include <fsm/bool.h>

#include "re_char_class.h"

/* Because we're building with -NDEBUG */
#define FAIL(s)					\
	do {					\
		fprintf(stderr, s);		\
		abort();			\
	} while(0)

/* Parse tree / Abstract syntax tree.
 * None of this should be exposed to the public API. */

enum ast_expr_type {
	AST_EXPR_EMPTY,
	AST_EXPR_CONCAT,
	AST_EXPR_ALT,
	AST_EXPR_LITERAL,
	AST_EXPR_ANY,
	AST_EXPR_MANY,
	AST_EXPR_KLEENE,
	AST_EXPR_PLUS,
	AST_EXPR_OPT,
	AST_EXPR_REPEATED,
	AST_EXPR_CHAR_CLASS,
	AST_EXPR_GROUP,
	AST_EXPR_FLAGS
};

#define AST_COUNT_UNBOUNDED ((unsigned)-1)
struct ast_count {
	unsigned low;
	struct ast_pos start;
	unsigned high;
	struct ast_pos end;
};

/*
 * The following regular expression fragments map to associated fsm states
= * as follows (transitions written in .fsm format):
 *
 *  ab    concat:      1 -> 3 "a"; 3 -> 2 "b";
 *  a|b   alt:         1 -> 2 "a"; 1 -> 2 "b";
 *  a     literal:     1 -> 1a; 2a -> 2;
 *  .     any:         1 -> 2 ?;
 *  (a)   group:       1 -> 1a; 2a -> 2;
 *  [abc] char-class:  1 -> 2 "a"; 1 -> 2 "b"; 1 -> 2 "c";
 */
struct ast_expr {
	enum ast_expr_type t;
	union {
		struct {
			struct ast_expr *l;<commit_msg>libre AST: remove enum values for removed node types.<commit_after>#ifndef RE_AST_H
#define RE_AST_H

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>

#include <re/re.h>

#include <fsm/fsm.h>
#include <fsm/bool.h>

#include "re_char_class.h"

/* Because we're building with -NDEBUG */
#define FAIL(s)					\
	do {					\
		fprintf(stderr, s);		\
		abort();			\
	} while(0)

/* Parse tree / Abstract syntax tree.
 * None of this should be exposed to the public API. */

enum ast_expr_type {
	AST_EXPR_EMPTY,
	AST_EXPR_CONCAT,
	AST_EXPR_ALT,
	AST_EXPR_LITERAL,
	AST_EXPR_ANY,
	AST_EXPR_MANY,
	AST_EXPR_REPEATED,
	AST_EXPR_CHAR_CLASS,
	AST_EXPR_GROUP,
	AST_EXPR_FLAGS
};

#define AST_COUNT_UNBOUNDED ((unsigned)-1)
struct ast_count {
	unsigned low;
	struct ast_pos start;
	unsigned high;
	struct ast_pos end;
};

/*
 * The following regular expression fragments map to associated fsm states
= * as follows (transitions written in .fsm format):
 *
 *  ab    concat:      1 -> 3 "a"; 3 -> 2 "b";
 *  a|b   alt:         1 -> 2 "a"; 1 -> 2 "b";
 *  a     literal:     1 -> 1a; 2a -> 2;
 *  .     any:         1 -> 2 ?;
 *  (a)   group:       1 -> 1a; 2a -> 2;
 *  [abc] char-class:  1 -> 2 "a"; 1 -> 2 "b"; 1 -> 2 "c";
 */
struct ast_expr {
	enum ast_expr_type t;
	union {
		struct {
			struct ast_expr *l;
{'fraction_non_alphanumeric': 0.1398095238095238, 'fraction_numerical': 0.019809523809523808, 'mean_word_length': 2.82798833819242, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2597008', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1091, 'n_words': 292}