<commit_before>
/* Parser implementation */

/* For a description, see the comments at end of this file */

/* XXX To do: error recovery */

#include "pgenheaders.h"
#include "assert.h"
#include "token.h"
#include "grammar.h"
#include "node.h"
#include "parser.h"
#include "errcode.h"


#ifdef Py_DEBUG
extern int Py_DebugFlag;
#define D(x) if (!Py_DebugFlag); else x
#else
#define D(x)
#endif


/* STACK DATA TYPE */

static void s_reset(stack *);

static void
s_reset(stack *s)
{
	s->s_top = &s->s_base[MAXSTACK];
}

#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])

static int
s_push(register stack *s, dfa *d, node *parent)
{
	register stackentry *top;
	if (s->s_top == s->s_base) {
		fprintf(stderr, "s_push: parser stack overflow\n");
		return E_NOMEM;
	}
	top = --s->s_top;
	top->s_dfa = d;
	top->s_parent = parent;
	top->s_state = 0;
	return 0;
}

#ifdef Py_DEBUG

static void
s_pop(register stack *s)
{
	if (s_empty(s))
		Py_FatalError("s_pop: parser stack underflow -- FATAL");
	s->s_top++;
}

#else /* !Py_DEBUG */

#define s_pop(s) (s)->s_top++

#endif


/* PARSER CREATION */

parser_state *
PyParser_New(grammar *g, int start)
{
	parser_state *ps;
	
	if (!g->g_accel)
		PyGrammar_AddAccelerators(g);
	ps = PyMem_NEW(parser_state, 1);
	if (ps == NULL)
		return NULL;
	ps->p_grammar = g;
	ps->p_generators = 0;
	ps->p_tree = PyNode_New(start);
	if (ps->p_tree == NULL) {
		PyMem_DEL(ps);
		return NULL;
	}
	s_reset(&ps->p_stack);
	(void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree);
	return ps;
}

void
PyParser_Delete(parser_state *ps)
{
	/* NB If you want to save the parse tree,
	   you must set p_tree to NULL before calling delparser! */
	PyNode_Free(ps->p_tree);
	PyMem_DEL(ps);
}


/* PARSER STACK OPERATIONS */

static int
shift(register stack *s, int type, char *str, int newstate, int lineno)
{
	int err;
	assert(!s_empty(s));
	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
	if (err)
		return err;
	s->s_top->s_state = newstate;
	return 0;
}

static int
push(register stack *s, int type, dfa *d, int newstate, int lineno)
{
	int err;
	register node *n;
	n = s->s_top->s_parent;
	assert(!s_empty(s));
	err = PyNode_AddChild(n, type, (char *)NULL, lineno);
	if (err)
		return err;
	s->s_top->s_state = newstate;
	return s_push(s, d, CHILD(n, NCH(n)-1));
}


/* PARSER PROPER */

static int
classify(parser_state *ps, int type, char *str)
{
	grammar *g = ps->p_grammar;
	register int n = g->g_ll.ll_nlabels;
	
	if (type == NAME) {
		register char *s = str;
		register label *l = g->g_ll.ll_label;
		register int i;
		for (i = n; i > 0; i--, l++) {
			if (l->lb_type == NAME && l->lb_str != NULL &&
					l->lb_str[0] == s[0] &&
					strcmp(l->lb_str, s) == 0) {
				if (!ps->p_generators &&
				    s[0] == 'y' &&
				    strcmp(s, "yield") == 0)
					break; /* not a keyword */
				D(printf("It's a keyword\n"));
				return n - i;
			}
		}
	}
	
	{
		register label *l = g->g_ll.ll_label;
		register int i;
		for (i = n; i > 0; i--, l++) {
			if (l->lb_type == type && l->lb_str == NULL) {
				D(printf("It's a token we know\n"));
				return n - i;
			}
		}
	}
	
	D(printf("Illegal token\n"));
	return -1;
}

static void
future_hack(parser_state *ps)
{
	node *n = ps->p_stack.s_top->s_parent;
	node *ch;

	if (strcmp(STR(CHILD(n, 0)), "from") != 0)
		return;
	ch = CHILD(n, 1);
	if (strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
		return;
	ch = CHILD(n, 3);
	if (NCH(ch) == 1 && strcmp(STR(CHILD(ch, 0)), "generators") == 0)
		ps->p_generators = 1;
}

int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
	          int lineno, int *expected_ret)<commit_msg>Fis SF bug #442647: not all forms of legal future statements were parsed correctly.  Now they are.<commit_after>
/* Parser implementation */

/* For a description, see the comments at end of this file */

/* XXX To do: error recovery */

#include "pgenheaders.h"
#include "assert.h"
#include "token.h"
#include "grammar.h"
#include "node.h"
#include "parser.h"
#include "errcode.h"


#ifdef Py_DEBUG
extern int Py_DebugFlag;
#define D(x) if (!Py_DebugFlag); else x
#else
#define D(x)
#endif


/* STACK DATA TYPE */

static void s_reset(stack *);

static void
s_reset(stack *s)
{
	s->s_top = &s->s_base[MAXSTACK];
}

#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])

static int
s_push(register stack *s, dfa *d, node *parent)
{
	register stackentry *top;
	if (s->s_top == s->s_base) {
		fprintf(stderr, "s_push: parser stack overflow\n");
		return E_NOMEM;
	}
	top = --s->s_top;
	top->s_dfa = d;
	top->s_parent = parent;
	top->s_state = 0;
	return 0;
}

#ifdef Py_DEBUG

static void
s_pop(register stack *s)
{
	if (s_empty(s))
		Py_FatalError("s_pop: parser stack underflow -- FATAL");
	s->s_top++;
}

#else /* !Py_DEBUG */

#define s_pop(s) (s)->s_top++

#endif


/* PARSER CREATION */

parser_state *
PyParser_New(grammar *g, int start)
{
	parser_state *ps;
	
	if (!g->g_accel)
		PyGrammar_AddAccelerators(g);
	ps = PyMem_NEW(parser_state, 1);
	if (ps == NULL)
		return NULL;
	ps->p_grammar = g;
	ps->p_generators = 0;
	ps->p_tree = PyNode_New(start);
	if (ps->p_tree == NULL) {
		PyMem_DEL(ps);
		return NULL;
	}
	s_reset(&ps->p_stack);
	(void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree);
	return ps;
}

void
PyParser_Delete(parser_state *ps)
{
	/* NB If you want to save the parse tree,
	   you must set p_tree to NULL before calling delparser! */
	PyNode_Free(ps->p_tree);
	PyMem_DEL(ps);
}


/* PARSER STACK OPERATIONS */

static int
shift(register stack *s, int type, char *str, int newstate, int lineno)
{
	int err;
	assert(!s_empty(s));
	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
	if (err)
		return err;
	s->s_top->s_state = newstate;
	return 0;
}

static int
push(register stack *s, int type, dfa *d, int newstate, int lineno)
{
	int err;
	register node *n;
	n = s->s_top->s_parent;
	assert(!s_empty(s));
	err = PyNode_AddChild(n, type, (char *)NULL, lineno);
	if (err)
		return err;
	s->s_top->s_state = newstate;
	return s_push(s, d, CHILD(n, NCH(n)-1));
}


/* PARSER PROPER */

static int
classify(parser_state *ps, int type, char *str)
{
	grammar *g = ps->p_grammar;
	register int n = g->g_ll.ll_nlabels;
	
	if (type == NAME) {
		register char *s = str;
		register label *l = g->g_ll.ll_label;
		register int i;
		for (i = n; i > 0; i--, l++) {
			if (l->lb_type == NAME && l->lb_str != NULL &&
					l->lb_str[0] == s[0] &&
					strcmp(l->lb_str, s) == 0) {
				if (!ps->p_generators &&
				    s[0] == 'y' &&
				    strcmp(s, "yield") == 0)
					break; /* not a keyword */
				D(printf("It's a keyword\n"));
				return n - i;
			}
		}
	}
	
	{
		register label *l = g->g_ll.ll_label;
		register int i;
		for (i = n; i > 0; i--, l++) {
			if (l->lb_type == type && l->lb_str == NULL) {
				D(printf("It's a token we know\n"));
				return n - i;
			}
		}
	}
	
	D(printf("Illegal token\n"));
	return -1;
}

static void
future_hack(parser_state *ps)
{
	node *n = ps->p_stack.s_top->s_parent;
	node *ch;
	int i;

	if (strcmp(STR(CHILD(n, 0)), "from") != 0)
		return;
	ch = CHILD(n, 1);
	if (strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
		return;
	for (i = 3; i < NCH(n); i += 2) {
		ch = CHILD(n, i);
		if (NCH(ch) >= 1 && TYPE(CHILD(ch, 0)) == NAME &&
		    strcmp(STR(CHILD(ch, 0)), "generators") == 0) {
			ps->p_generators = 1;
			break;
		}
	}
}

int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
	          int lineno, int *expected_ret)
{'fraction_non_alphanumeric': 0.17750133761369716, 'fraction_numerical': 0.00749063670411985, 'mean_word_length': 3.469216975493126, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2524926', 'n_tokens_mistral': 3408, 'n_tokens_neox': 3172, 'n_words': 818}