<commit_before>#include <stdlib.h>

#include "post_fmt3_ast.h"
#include "error.h"

static struct astnode *__cvt_ptnode(struct ptnode *pn)
{
	struct astnode *an;

	switch (pn->type) {
		case PT_STR:
			an = astnode_new_str(pn->u.str);
			break;
		case PT_CHAR:
			an = astnode_new_char(pn->u.ch.ch, pn->u.ch.len);
			break;
		case PT_MATH:
			an = astnode_new_math(pn->u.str);
			break;
		default:
			an = NULL;
			ASSERT(0);
			break;
	}

	ASSERT(an);

	return an;
}

static int __cvt(struct ast *ast, struct list_head *nodes)
{<commit_msg>post fmt3: describe the parse tree to AST conversion process<commit_after>#include <stdlib.h>

#include "post_fmt3_ast.h"
#include "error.h"

/*
 * Converting the parse tree is relatively easy, but it does take some
 * effort.
 *
 * First of all, the parse tree is a nested stream of tokens.  It is not a
 * simple linear stream that the lexer spits out, but rather certain sets of
 * tokens are nested within parse nodes.  For example, 'foo {bar} baz' will
 * turn into a PT_STR ("foo"), PT_STR (space), PT_OPT_MAN, PT_STR (space),
 * PT_STR ("baz").  The PT_OPT_MAN's interpretation will be context
 * sensitive.  The code in this file is responsible for the conversion.  In
 * this case, since PT_OPT_MAN is not following a command, it should be
 * treated as a string of the contents (i.e., the {} get silently dropped).
 *
 * There are several different bits of context that we need to keep track
 * of.  In no particular order, they are:
 *
 *  1) command invocation - we need to know the most recent command
 *     invocation that is affecting the current parse node.  For example, in
 *     "\mbox{foo}{bar}", the "{foo}" node is affected by the \mbox command
 *     but "{bar}" is not.  The parse node stream will have a sequence of
 *     PT_CMD, PT_OPT_MAN, and PT_OPT_MAN.
 *
 *     The command table needs to inform us about how many optional and
 *     mandatory arguments to expect.
 *
 *  2) environments - consider this example: "\begin{foo}xyzzy\end{foo}".
 *     The parse node stream will simply contain PT_CMD, PT_OPT_MAN, PT_STR,
 *     PT_CMD, and PT_OPT_MAN.  Obiously, the PT_CMDs affect the immediate
 *     PT_OPT_MANs (see point #1).  However, as part of the AST
 *     construction, we want to turn the \begin & \end pair into an AST_ENV
 *     with all the in-between stuff underneath it.
 *
 *  3) environment nesting - environments can be nested, but the \begin and
 *     \end must happen at the same nesting level and commands cannot start
 *     outside of an environment and end inside or vice versa.
 */

static struct astnode *__cvt_ptnode(struct ptnode *pn)
{
	struct astnode *an;

	switch (pn->type) {
		case PT_STR:
			an = astnode_new_str(pn->u.str);
			break;
		case PT_CHAR:
			an = astnode_new_char(pn->u.ch.ch, pn->u.ch.len);
			break;
		case PT_MATH:
			an = astnode_new_math(pn->u.str);
			break;
		default:
			an = NULL;
			ASSERT(0);
			break;
	}

	ASSERT(an);

	return an;
}

static int __cvt(struct ast *ast, struct list_head *nodes)
{
{'fraction_non_alphanumeric': 0.09976874793524942, 'fraction_numerical': 0.002973240832507433, 'mean_word_length': 3.331902718168813, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6151323', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1003, 'n_words': 388}