<commit_before>#include "js.h"
#include "js-parse.h"
#include "js-ast.h"

#define A1(x,a)		jsP_newnode(J, x, a, 0, 0, 0)
#define A2(x,a,b)	jsP_newnode(J, x, a, b, 0, 0)
#define A3(x,a,b,c)	jsP_newnode(J, x, a, b, c, 0)

#define LIST(h,t)	jsP_newnode(J, AST_LIST, h, t, 0, 0);
#define HEAD(h)		jsP_newnode(J, AST_LIST, h, 0, 0, 0);
#define TAIL(t)		jsP_newnode(J, AST_LIST, 0, t, 0, 0);

#define ID(s)		jsP_newsnode(J, AST_IDENTIFIER, s)

#define EXP0(x)		jsP_newnode(J, EXP_ ## x, 0, 0, 0, 0)
#define EXP1(x,a)	jsP_newnode(J, EXP_ ## x, a, 0, 0, 0)
#define EXP2(x,a,b)	jsP_newnode(J, EXP_ ## x, a, b, 0, 0)
#define EXP3(x,a,b,c)	jsP_newnode(J, EXP_ ## x, a, b, c, 0)

#define STM1(x,a)	jsP_newnode(J, STM_ ## x, a, 0, 0, 0)
#define STM2(x,a,b)	jsP_newnode(J, STM_ ## x, a, b, 0, 0)
#define STM3(x,a,b,c)	jsP_newnode(J, STM_ ## x, a, b, c, 0)
#define STM4(x,a,b,c,d)	jsP_newnode(J, STM_ ## x, a, b, c, d)

static js_Ast *expression(js_State *J, int notin);
static js_Ast *assignment(js_State *J, int notin);
static js_Ast *memberexp(js_State *J);
static js_Ast *block(js_State *J);

static const char *tokenstring[] = {
	"(end-of-file)",
	"'\\x01'", "'\\x02'", "'\\x03'", "'\\x04'", "'\\x05'", "'\\x06'", "'\\x07'",
	"'\\x08'", "'\\x09'", "'\\x0A'", "'\\x0B'", "'\\x0C'", "'\\x0D'", "'\\x0E'", "'\\x0F'",
	"'\\x10'", "'\\x11'", "'\\x12'", "'\\x13'", "'\\x14'", "'\\x15'", "'\\x16'", "'\\x17'",
	"'\\x18'", "'\\x19'", "'\\x1A'", "'\\x1B'", "'\\x1C'", "'\\x1D'", "'\\x1E'", "'\\x1F'",
	"' '", "'!'", "'\"'", "'#'", "'$'", "'%'", "'&'", "'\\''",
	"'('", "')'", "'*'", "'+'", "','", "'-'", "'.'", "'/'",
	"'0'", "'1'", "'2'", "'3'", "'4'", "'5'", "'6'", "'7'",
	"'8'", "'9'", "':'", "';'", "'<'", "'='", "'>'", "'?'",
	"'@'", "'A'", "'B'", "'C'", "'D'", "'E'", "'F'", "'G'",
	"'H'", "'I'", "'J'", "'K'", "'L'", "'M'", "'N'", "'O'",
	"'P'", "'Q'", "'R'", "'S'", "'T'", "'U'", "'V'", "'W'",
	"'X'", "'Y'", "'Z'", "'['", "'\'", "']'", "'^'", "'_'",
	"'`'", "'a'", "'b'", "'c'", "'d'", "'e'", "'f'", "'g'",
	"'h'", "'i'", "'j'", "'k'", "'l'", "'m'", "'n'", "'o'",
	"'p'", "'q'", "'r'", "'s'", "'t'", "'u'", "'v'", "'w'",
	"'x'", "'y'", "'z'", "'{'", "'|'", "'}'", "'~'", "'\\x7F'",

	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,

	"(identifier)", "(number)", "(string)", "(regexp)",

	"'<='", "'>='", "'=='", "'!='", "'==='", "'!=='",
	"'<<'", "'>>'", "'>>>'", "'&&'", "'||'",
	"'+='", "'-='", "'*='", "'/='", "'%='",
	"'<<='", "'>>='", "'>>>='", "'&='", "'|='", "'^='",
	"'++'", "'--'",

	"'break'", "'case'", "'catch'", "'continue'", "'debugger'",
	"'default'", "'delete'", "'do'", "'else'", "'false'", "'finally'", "'for'",
	"'function'", "'if'", "'in'", "'instanceof'", "'new'", "'null'", "'return'",
	"'switch'", "'this'", "'throw'", "'true'", "'try'", "'typeof'", "'var'",
	"'void'", "'while'", "'with'",
};

static void next(js_State *J)
{
	J->lookahead = jsP_lex(J);
}

static int accept(js_State *J, int t)
{
	if (J->lookahead == t) {
		next(J);
		return 1;
	}
	return 0;
}

static void expect(js_State *J, int t)
{
	if (accept(J, t))
		return;
	jsP_error(J, "unexpected token: %s (expected %s)", tokenstring[J->lookahead], tokenstring[t]);
}

static void semicolon(js_State *J)
{
	if (J->lookahead == ';') {
		next(J);
		return;
	}
	if (J->newline || J->lookahead == '}' || J->lookahead == 0)
		return;
	jsP_error(J, "unexpected token: %s (expected ';')", tokenstring[J->lookahead]);
}

static js_Ast *identifier(js_State *J)
{
	if (J->lookahead == TK_IDENTIFIER) {
		js_Ast *a = ID(J->yytext);
		next(J);
		return a;
	}
	jsP_error(J, "unexpected token: %s (expected identifier)", tokenstring[J->lookahead]);
	return NULL;
}

static js_Ast *identifiername(js_State *J)
{
	if (J->lookahead == TK_IDENTIFIER || J->lookahead >= TK_BREAK) {
		js_Ast *a = ID(J->yytext);
		next(J);
		return a;
	}
	jsP_error(J, "unexpected token: %s (expected identifier or keyword)", tokenstring[J->lookahead]);
	return NULL;
}

static js_Ast *arguments(js_State *J)
{
	js_Ast *head, *node;

	if (J->lookahead == ')')
		return NULL;

	node = assignment(J, 0);
	head = HEAD(node);
	while (accept(J, ',')) {
		node = assignment(J, 0);
		head = LIST(head, node);
	}
	return head;
}

static js_Ast *arrayliteral(js_State *J)
{
	js_Ast *head, *node;

	while (J->lookahead == ',')
		next(J);

	if (J->lookahead == ']')
		return NULL;

	node = assignment(J, 0);
	head = HEAD(node);
	while (accept(J, ',')) {
		while (J->lookahead == ',')
			next(J);
		if (J->lookahead == ']')
			break;
		node = assignment(J, 0);
		head = LIST(head, node);
	}
	return head;
}

static js_Ast *propname(js_State *J)
{
	js_Ast *name;
	if (J->lookahead == TK_NUMBER) {
		name = jsP_newnnode(J, AST_NUMBER, J->yynumber);
		next(J);
	} else if (J->lookahead == TK_STRING) {
		name = jsP_newsnode(J, AST_NUMBER, J->yytext);
		next(J);
	} else {
		name = identifiername(J);
	}
	return name;
}

static js_Ast *propassign(js_State *J)
{
	js_Ast *name, *value, *arg, *body;

	if (J->lookahead == TK_IDENTIFIER && !strcmp(J->yytext, "get")) {
		next(J);
		name = propname(J);
		expect(J, '(');
		expect(J, ')');
		body = block(J);
		return EXP2(PROP_GET, name, body);
	}

	if (J->lookahead == TK_IDENTIFIER && !strcmp(J->yytext, "set")) {
		next(J);
		name = propname(J);
		expect(J, '(');
		arg = identifier(J);<commit_msg>Fix typo. Remove ID macro.<commit_after>#include "js.h"
#include "js-parse.h"
#include "js-ast.h"

#define A1(x,a)		jsP_newnode(J, x, a, 0, 0, 0)
#define A2(x,a,b)	jsP_newnode(J, x, a, b, 0, 0)
#define A3(x,a,b,c)	jsP_newnode(J, x, a, b, c, 0)

#define LIST(h,t)	jsP_newnode(J, AST_LIST, h, t, 0, 0);
#define HEAD(h)		jsP_newnode(J, AST_LIST, h, 0, 0, 0);
#define TAIL(t)		jsP_newnode(J, AST_LIST, 0, t, 0, 0);

#define EXP0(x)		jsP_newnode(J, EXP_ ## x, 0, 0, 0, 0)
#define EXP1(x,a)	jsP_newnode(J, EXP_ ## x, a, 0, 0, 0)
#define EXP2(x,a,b)	jsP_newnode(J, EXP_ ## x, a, b, 0, 0)
#define EXP3(x,a,b,c)	jsP_newnode(J, EXP_ ## x, a, b, c, 0)

#define STM1(x,a)	jsP_newnode(J, STM_ ## x, a, 0, 0, 0)
#define STM2(x,a,b)	jsP_newnode(J, STM_ ## x, a, b, 0, 0)
#define STM3(x,a,b,c)	jsP_newnode(J, STM_ ## x, a, b, c, 0)
#define STM4(x,a,b,c,d)	jsP_newnode(J, STM_ ## x, a, b, c, d)

static js_Ast *expression(js_State *J, int notin);
static js_Ast *assignment(js_State *J, int notin);
static js_Ast *memberexp(js_State *J);
static js_Ast *block(js_State *J);

static const char *tokenstring[] = {
	"(end-of-file)",
	"'\\x01'", "'\\x02'", "'\\x03'", "'\\x04'", "'\\x05'", "'\\x06'", "'\\x07'",
	"'\\x08'", "'\\x09'", "'\\x0A'", "'\\x0B'", "'\\x0C'", "'\\x0D'", "'\\x0E'", "'\\x0F'",
	"'\\x10'", "'\\x11'", "'\\x12'", "'\\x13'", "'\\x14'", "'\\x15'", "'\\x16'", "'\\x17'",
	"'\\x18'", "'\\x19'", "'\\x1A'", "'\\x1B'", "'\\x1C'", "'\\x1D'", "'\\x1E'", "'\\x1F'",
	"' '", "'!'", "'\"'", "'#'", "'$'", "'%'", "'&'", "'\\''",
	"'('", "')'", "'*'", "'+'", "','", "'-'", "'.'", "'/'",
	"'0'", "'1'", "'2'", "'3'", "'4'", "'5'", "'6'", "'7'",
	"'8'", "'9'", "':'", "';'", "'<'", "'='", "'>'", "'?'",
	"'@'", "'A'", "'B'", "'C'", "'D'", "'E'", "'F'", "'G'",
	"'H'", "'I'", "'J'", "'K'", "'L'", "'M'", "'N'", "'O'",
	"'P'", "'Q'", "'R'", "'S'", "'T'", "'U'", "'V'", "'W'",
	"'X'", "'Y'", "'Z'", "'['", "'\'", "']'", "'^'", "'_'",
	"'`'", "'a'", "'b'", "'c'", "'d'", "'e'", "'f'", "'g'",
	"'h'", "'i'", "'j'", "'k'", "'l'", "'m'", "'n'", "'o'",
	"'p'", "'q'", "'r'", "'s'", "'t'", "'u'", "'v'", "'w'",
	"'x'", "'y'", "'z'", "'{'", "'|'", "'}'", "'~'", "'\\x7F'",

	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
	0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,

	"(identifier)", "(number)", "(string)", "(regexp)",

	"'<='", "'>='", "'=='", "'!='", "'==='", "'!=='",
	"'<<'", "'>>'", "'>>>'", "'&&'", "'||'",
	"'+='", "'-='", "'*='", "'/='", "'%='",
	"'<<='", "'>>='", "'>>>='", "'&='", "'|='", "'^='",
	"'++'", "'--'",

	"'break'", "'case'", "'catch'", "'continue'", "'debugger'",
	"'default'", "'delete'", "'do'", "'else'", "'false'", "'finally'", "'for'",
	"'function'", "'if'", "'in'", "'instanceof'", "'new'", "'null'", "'return'",
	"'switch'", "'this'", "'throw'", "'true'", "'try'", "'typeof'", "'var'",
	"'void'", "'while'", "'with'",
};

static void next(js_State *J)
{
	J->lookahead = jsP_lex(J);
}

static int accept(js_State *J, int t)
{
	if (J->lookahead == t) {
		next(J);
		return 1;
	}
	return 0;
}

static void expect(js_State *J, int t)
{
	if (accept(J, t))
		return;
	jsP_error(J, "unexpected token: %s (expected %s)", tokenstring[J->lookahead], tokenstring[t]);
}

static void semicolon(js_State *J)
{
	if (J->lookahead == ';') {
		next(J);
		return;
	}
	if (J->newline || J->lookahead == '}' || J->lookahead == 0)
		return;
	jsP_error(J, "unexpected token: %s (expected ';')", tokenstring[J->lookahead]);
}

static js_Ast *identifier(js_State *J)
{
	if (J->lookahead == TK_IDENTIFIER) {
		js_Ast *a = jsP_newsnode(J, AST_IDENTIFIER, J->yytext);
		next(J);
		return a;
	}
	jsP_error(J, "unexpected token: %s (expected identifier)", tokenstring[J->lookahead]);
	return NULL;
}

static js_Ast *identifiername(js_State *J)
{
	if (J->lookahead == TK_IDENTIFIER || J->lookahead >= TK_BREAK) {
		js_Ast *a = jsP_newsnode(J, AST_IDENTIFIER, J->yytext);
		next(J);
		return a;
	}
	jsP_error(J, "unexpected token: %s (expected identifier or keyword)", tokenstring[J->lookahead]);
	return NULL;
}

static js_Ast *arguments(js_State *J)
{
	js_Ast *head, *node;

	if (J->lookahead == ')')
		return NULL;

	node = assignment(J, 0);
	head = HEAD(node);
	while (accept(J, ',')) {
		node = assignment(J, 0);
		head = LIST(head, node);
	}
	return head;
}

static js_Ast *arrayliteral(js_State *J)
{
	js_Ast *head, *node;

	while (J->lookahead == ',')
		next(J);

	if (J->lookahead == ']')
		return NULL;

	node = assignment(J, 0);
	head = HEAD(node);
	while (accept(J, ',')) {
		while (J->lookahead == ',')
			next(J);
		if (J->lookahead == ']')
			break;
		node = assignment(J, 0);
		head = LIST(head, node);
	}
	return head;
}

static js_Ast *propname(js_State *J)
{
	js_Ast *name;
	if (J->lookahead == TK_NUMBER) {
		name = jsP_newnnode(J, AST_NUMBER, J->yynumber);
		next(J);
	} else if (J->lookahead == TK_STRING) {
		name = jsP_newsnode(J, AST_STRING, J->yytext);
		next(J);
	} else {
		name = identifiername(J);
	}
	return name;
}

static js_Ast *propassign(js_State *J)
{
	js_Ast *name, *value, *arg, *body;

	if (J->lookahead == TK_IDENTIFIER && !strcmp(J->yytext, "get")) {
		next(J);
		name = propname(J);
		expect(J, '(');
		expect(J, ')');
		body = block(J);
		return EXP2(PROP_GET, name, body);
	}

	if (J->lookahead == TK_IDENTIFIER && !strcmp(J->yytext, "set")) {
		next(J);
		name = propname(J);
		expect(J, '(');
		arg = identifier(J);
{'fraction_non_alphanumeric': 0.34952070898896725, 'fraction_numerical': 0.04304575872671369, 'mean_word_length': 4.485615079365079, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6902647', 'n_tokens_mistral': 5736, 'n_tokens_neox': 5596, 'n_words': 1094}