<commit_before>/*
 * This file is part of Hubbub.
 * Licensed under the MIT License,
 *                http://www.opensource.org/licenses/mit-license.php
 * Copyright 2007-8 <NAME> <<EMAIL>>
 */

#ifndef hubbub_parser_h_
#define hubbub_parser_h_

#include <inttypes.h>

#include <hubbub/errors.h>
#include <hubbub/functypes.h>
#include <hubbub/tree.h>
#include <hubbub/types.h>

typedef struct hubbub_parser hubbub_parser;

/**
 * Hubbub parser option types
 */
typedef enum hubbub_parser_opttype {
	HUBBUB_PARSER_TOKEN_HANDLER,
	HUBBUB_PARSER_ERROR_HANDLER,
	HUBBUB_PARSER_CONTENT_MODEL,
	HUBBUB_PARSER_TREE_HANDLER,
	HUBBUB_PARSER_DOCUMENT_NODE,
} hubbub_parser_opttype;

/**
 * Hubbub parser option parameters
 */
typedef union hubbub_parser_optparams {
	struct {
		hubbub_token_handler handler;
		void *pw;
	} token_handler;

	struct {
		hubbub_error_handler handler;
		void *pw;
	} error_handler;

	struct {
		hubbub_content_model model;
	} content_model;

	hubbub_tree_handler *tree_handler;

	void *document_node;
} hubbub_parser_optparams;

/* Create a hubbub parser */
hubbub_parser *hubbub_parser_create(const char *enc,
		hubbub_alloc alloc, void *pw);
/* Destroy a hubbub parser */
void hubbub_parser_destroy(hubbub_parser *parser);

/* Configure a hubbub parser */
hubbub_error hubbub_parser_setopt(hubbub_parser *parser,
		hubbub_parser_opttype type,
		hubbub_parser_optparams *params);

/* Pass a chunk of data to a hubbub parser for parsing */
/* This data is encoded in the input charset */
hubbub_error hubbub_parser_parse_chunk(hubbub_parser *parser,
		uint8_t *data, size_t len);
/* Pass a chunk of extraneous data to a hubbub parser for parsing */
/* This data is UTF-8 encoded */
hubbub_error hubbub_parser_parse_extraneous_chunk(hubbub_parser *parser,
		uint8_t *data, size_t len);
/* Inform the parser that the last chunk of data has been parsed */
hubbub_error hubbub_parser_completed(hubbub_parser *parser);

/* Read the document charset */
const char *hubbub_parser_read_charset(hubbub_parser *parser,
		hubbub_charset_source *source);

/* Claim ownership of the document buffer */
hubbub_error hubbub_parser_claim_buffer(hubbub_parser *parser,
		uint8_t **buffer, size_t *len);

#endif

<commit_msg>Er yes. Let's commit the headers, too, shall we?<commit_after>/*
 * This file is part of Hubbub.
 * Licensed under the MIT License,
 *                http://www.opensource.org/licenses/mit-license.php
 * Copyright 2007-8 <NAME> <<EMAIL>>
 */

#ifndef hubbub_parser_h_
#define hubbub_parser_h_

#include <inttypes.h>

#include <hubbub/errors.h>
#include <hubbub/functypes.h>
#include <hubbub/tree.h>
#include <hubbub/types.h>

typedef struct hubbub_parser hubbub_parser;

/**
 * Hubbub parser option types
 */
typedef enum hubbub_parser_opttype {
	HUBBUB_PARSER_TOKEN_HANDLER,
	HUBBUB_PARSER_ERROR_HANDLER,
	HUBBUB_PARSER_CONTENT_MODEL,
	HUBBUB_PARSER_TREE_HANDLER,
	HUBBUB_PARSER_DOCUMENT_NODE,
	HUBBUB_PARSER_ENABLE_SCRIPTING,
} hubbub_parser_opttype;

/**
 * Hubbub parser option parameters
 */
typedef union hubbub_parser_optparams {
	struct {
		hubbub_token_handler handler;
		void *pw;
	} token_handler;

	struct {
		hubbub_error_handler handler;
		void *pw;
	} error_handler;

	struct {
		hubbub_content_model model;
	} content_model;

	hubbub_tree_handler *tree_handler;

	void *document_node;

	bool enable_scripting;
} hubbub_parser_optparams;

/* Create a hubbub parser */
hubbub_parser *hubbub_parser_create(const char *enc,
		hubbub_alloc alloc, void *pw);
/* Destroy a hubbub parser */
void hubbub_parser_destroy(hubbub_parser *parser);

/* Configure a hubbub parser */
hubbub_error hubbub_parser_setopt(hubbub_parser *parser,
		hubbub_parser_opttype type,
		hubbub_parser_optparams *params);

/* Pass a chunk of data to a hubbub parser for parsing */
/* This data is encoded in the input charset */
hubbub_error hubbub_parser_parse_chunk(hubbub_parser *parser,
		uint8_t *data, size_t len);
/* Pass a chunk of extraneous data to a hubbub parser for parsing */
/* This data is UTF-8 encoded */
hubbub_error hubbub_parser_parse_extraneous_chunk(hubbub_parser *parser,
		uint8_t *data, size_t len);
/* Inform the parser that the last chunk of data has been parsed */
hubbub_error hubbub_parser_completed(hubbub_parser *parser);

/* Read the document charset */
const char *hubbub_parser_read_charset(hubbub_parser *parser,
		hubbub_charset_source *source);

/* Claim ownership of the document buffer */
hubbub_error hubbub_parser_claim_buffer(hubbub_parser *parser,
		uint8_t **buffer, size_t *len);

#endif


{'fraction_non_alphanumeric': 0.08687046404222565, 'fraction_numerical': 0.003958654057620409, 'mean_word_length': 5.179347826086956, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515197', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1679, 'n_words': 401}