<commit_before>/*
 * (c) copyright 1983 by the Vrije Universiteit, Amsterdam, The Netherlands.
 *
 *          This product is part of the Amsterdam Compiler Kit.
 *
 * Permission to use, sell, duplicate or disclose this software must be
 * obtained in writing. Requests for such permissions may be sent to
 *
 *      Dr. <NAME>
 *      Wiskundig Seminarium
 *      Vrije Universiteit
 *      Postbox 7161
 *      1007 MC Amsterdam
 *      The Netherlands
 *
 */

/*
 *  L L G E N
 *
 *  An Extended LL(1) Parser Generator
 *
 *  Author : <NAME>
 */

/*
 * tokens.g
 * Defines the tokens for the grammar of LLgen.
 * The lexical analyser and LLmes are also included here. 
 */

{
# include "types.h"
# include "io.h"
# include "tunable.h"
# include "extern.h"
# include "assert.h"

# ifndef NORCSID
static string	rcsidc = "$Header$";
# endif

/* Here are defined : */
extern int	scanner();
extern 		LLmessage();
extern int	input();
extern		unput();
extern		skipcomment();
STATIC		linedirective();
STATIC string	cpy();
STATIC string	vallookup();
}

/* Classes */

%token  C_IDENT ;	/* lextoken.t_string contains the identifier read */
%token	C_NUMBER ;	/* lextoken.t_num contains the number read */
%token	C_LITERAL ;	/* lextoken.t_string contains the literal read */

/* Keywords */

%token	C_TOKEN ;
%token	C_START ;
%token	C_IF ;
%token	C_WHILE ;
%token	C_PERSISTENT ;
%token	C_FIRST ;
%token	C_LEXICAL ;
%token	C_AVOID ;
%token	C_PREFER ;
%token	C_DEFAULT ;

%lexical scanner ;

{

/*
 * Structure for a keyword
 */

struct keyword {
	string	w_word;
	int	w_value;
};

/*
 * The list of keywords, the most often used keywords come first.
 * Linear search is used, as there are not many keywords
 */

static struct keyword resword[] = {
	{ "token",	C_TOKEN	},
	{ "avoid",	C_AVOID	},
	{ "prefer",	C_PREFER	},
	{ "persistent", C_PERSISTENT	},
	{ "default",	C_DEFAULT	},
	{ "if",		C_IF	},
	{ "while",	C_WHILE	},
	{ "first",	C_FIRST	},
	{ "start",	C_START	},
	{ "lexical",	C_LEXICAL	},
	{ 0,		0	}
};

static t_token	savedtok;	/* to save lextoken in case of an insertion */
static  int	nostartline;	/* = 0 if at the start of a line */

scanner() {
	/*
	 * Lexical analyser, what else
	 */
	register	ch;		/* Current char */
	register	i;
	register	reserved = 0;	/* reserved word? */
	int		last;		/* Char before current char */

	if (savedtok.t_tokno) {	/*
				 * A token has been inserted.
				 * Now deliver the last lextoken again
				 */
		lextoken = savedtok;
		savedtok.t_tokno = 0;
		return lextoken.t_tokno;
	}
	for (;;) {	/*
			 * First, skip space, comments, line directives, etc
			 */
		do	ch = input();
		while(isspace(ch));
		if (ch == '/') skipcomment(0);
		else if (ch == '#' && !nostartline) linedirective();
		else break;
	}
	/*
	 * Now we have a first character of a token
	 */
	switch(ch) {
	  case EOF :
	 	 return EOF;
	  case '\'':	/*
			 * Literal, put it in ltext
			 */
		i = 0;
		for (;;) {
			last = ch;
			ch = input();
			if (ch == '\n' || ch == EOF) {
				error(linecount,"missing '");
				break;
			}
			if (ch == '\'' && last != '\\') break;
			ltext[i] = ch;
			if (i < LTEXTSZ - 1) ++i;
		}
		ltext[i] = '\0';
		lextoken.t_string = ltext;
		return C_LITERAL;
	  case '%' :	/*
			 * Start of a reserved word
			 */
		reserved = 1;
		ch = input();
		/* Fall through */
	  default  :
		i = 0;
		if (isdigit(ch)) {
			if (reserved) {
				error(linecount," A reserved number ?");
			}
			while (isdigit(ch)) {
				i = 10 * i + (ch - '0');
				ch= input();
			}
			lextoken.t_num = i;
			unput(ch);
			return C_NUMBER;
		}
		if (isalpha(ch) || ch == '_') {
			do {
				if (reserved && isupper(ch)) ch += 'a' - 'A';
				ltext[i] = ch;
				if (i < LTEXTSZ - 1) ++i;
				ch = input();
			} while (isalnum(ch) || ch == '_');
		} else	return ch;
		unput(ch);
	}
	ltext[i] = '\0';
	if (reserved) {	/*
			 * Now search for the keyword
			 */
		register struct keyword *w;

		w = resword;
		while (w->w_word) {
			if (! strcmp(ltext,w->w_word)) {
				/*
				 * Found it. Return token number.
				 */
				return w->w_value;
			}
			w++;
		}
		error(linecount,"illegal reserved word");
	}
	lextoken.t_string = ltext;
	return C_IDENT;
}

static int	backupc;	/* for unput() */
static int	nonline;	/* = 1 if last char read was a newline */

input() {
	/*
	 * Low level input routine, used by all other input routines
	 */
	register	c;
	register FILE	*f;

        if(backupc) {	/*
			 * Last char was "unput()". Deliver it again
			 */
		c = backupc;
		backupc = 0;
                return c;
	}
	f = finput;
	if ((c = getc(f)) == EOF) return c;
	nostartline = 1;
	if (!nonline) {
		linecount++;
		nostartline = 0;
		nonline = 1;
	}
	if (c == ' ' || c == '\t') {	/*
					 * Deliver space, but only once
					 */
		do	c = getc(f);
		while (c == ' ' || c == '\t');
		ungetc(c,f);
		return ' ';
	}
	if (c == '\n') nonline = 0;
	return c;
}

unput(c) {
	/*
	 * "unread" c
	 */
	backupc = c;
}

skipcomment(flag) {
	/*
	 * Skip comment. If flag != 0, the comment is inside a fragment
	 * of C-code, so the newlines in it must be copied to enable the
	 * C-compiler to keep a correct line count
	 */<commit_msg>The input routine used to skip space. This "feature" has been removed because it is also used to read strings, in which space is significant.<commit_after>/*
 * (c) copyright 1983 by the Vrije Universiteit, Amsterdam, The Netherlands.
 *
 *          This product is part of the Amsterdam Compiler Kit.
 *
 * Permission to use, sell, duplicate or disclose this software must be
 * obtained in writing. Requests for such permissions may be sent to
 *
 *      Dr. <NAME>
 *      Wiskundig Seminarium
 *      Vrije Universiteit
 *      Postbox 7161
 *      1007 MC Amsterdam
 *      The Netherlands
 *
 */

/*
 *  L L G E N
 *
 *  An Extended LL(1) Parser Generator
 *
 *  Author : <NAME>
 */

/*
 * tokens.g
 * Defines the tokens for the grammar of LLgen.
 * The lexical analyser and LLmes are also included here. 
 */

{
# include "types.h"
# include "io.h"
# include "tunable.h"
# include "extern.h"
# include "assert.h"

# ifndef NORCSID
static string	rcsidc = "$Header$";
# endif

/* Here are defined : */
extern int	scanner();
extern 		LLmessage();
extern int	input();
extern		unput();
extern		skipcomment();
STATIC		linedirective();
STATIC string	cpy();
STATIC string	vallookup();
}

/* Classes */

%token  C_IDENT ;	/* lextoken.t_string contains the identifier read */
%token	C_NUMBER ;	/* lextoken.t_num contains the number read */
%token	C_LITERAL ;	/* lextoken.t_string contains the literal read */

/* Keywords */

%token	C_TOKEN ;
%token	C_START ;
%token	C_IF ;
%token	C_WHILE ;
%token	C_PERSISTENT ;
%token	C_FIRST ;
%token	C_LEXICAL ;
%token	C_AVOID ;
%token	C_PREFER ;
%token	C_DEFAULT ;

%lexical scanner ;

{

/*
 * Structure for a keyword
 */

struct keyword {
	string	w_word;
	int	w_value;
};

/*
 * The list of keywords, the most often used keywords come first.
 * Linear search is used, as there are not many keywords
 */

static struct keyword resword[] = {
	{ "token",	C_TOKEN	},
	{ "avoid",	C_AVOID	},
	{ "prefer",	C_PREFER	},
	{ "persistent", C_PERSISTENT	},
	{ "default",	C_DEFAULT	},
	{ "if",		C_IF	},
	{ "while",	C_WHILE	},
	{ "first",	C_FIRST	},
	{ "start",	C_START	},
	{ "lexical",	C_LEXICAL	},
	{ 0,		0	}
};

static t_token	savedtok;	/* to save lextoken in case of an insertion */
static  int	nostartline;	/* = 0 if at the start of a line */

scanner() {
	/*
	 * Lexical analyser, what else
	 */
	register	ch;		/* Current char */
	register	i;
	register	reserved = 0;	/* reserved word? */
	int		last;		/* Char before current char */

	if (savedtok.t_tokno) {	/*
				 * A token has been inserted.
				 * Now deliver the last lextoken again
				 */
		lextoken = savedtok;
		savedtok.t_tokno = 0;
		return lextoken.t_tokno;
	}
	for (;;) {	/*
			 * First, skip space, comments, line directives, etc
			 */
		do	ch = input();
		while(isspace(ch));
		if (ch == '/') skipcomment(0);
		else if (ch == '#' && !nostartline) linedirective();
		else break;
	}
	/*
	 * Now we have a first character of a token
	 */
	switch(ch) {
	  case EOF :
	 	 return EOF;
	  case '\'':	/*
			 * Literal, put it in ltext
			 */
		i = 0;
		for (;;) {
			last = ch;
			ch = input();
			if (ch == '\n' || ch == EOF) {
				error(linecount,"missing '");
				break;
			}
			if (ch == '\'' && last != '\\') break;
			ltext[i] = ch;
			if (i < LTEXTSZ - 1) ++i;
		}
		ltext[i] = '\0';
		lextoken.t_string = ltext;
		return C_LITERAL;
	  case '%' :	/*
			 * Start of a reserved word
			 */
		reserved = 1;
		ch = input();
		/* Fall through */
	  default  :
		i = 0;
		if (isdigit(ch)) {
			if (reserved) {
				error(linecount," A reserved number ?");
			}
			while (isdigit(ch)) {
				i = 10 * i + (ch - '0');
				ch= input();
			}
			lextoken.t_num = i;
			unput(ch);
			return C_NUMBER;
		}
		if (isalpha(ch) || ch == '_') {
			do {
				if (reserved && isupper(ch)) ch += 'a' - 'A';
				ltext[i] = ch;
				if (i < LTEXTSZ - 1) ++i;
				ch = input();
			} while (isalnum(ch) || ch == '_');
		} else	return ch;
		unput(ch);
	}
	ltext[i] = '\0';
	if (reserved) {	/*
			 * Now search for the keyword
			 */
		register struct keyword *w;

		w = resword;
		while (w->w_word) {
			if (! strcmp(ltext,w->w_word)) {
				/*
				 * Found it. Return token number.
				 */
				return w->w_value;
			}
			w++;
		}
		error(linecount,"illegal reserved word");
	}
	lextoken.t_string = ltext;
	return C_IDENT;
}

static int	backupc;	/* for unput() */
static int	nonline;	/* = 1 if last char read was a newline */

input() {
	/*
	 * Low level input routine, used by all other input routines
	 */
	register	c;
	register FILE	*f;

        if(backupc) {	/*
			 * Last char was "unput()". Deliver it again
			 */
		c = backupc;
		backupc = 0;
                return c;
	}
	f = finput;
	if ((c = getc(f)) == EOF) return c;
	nostartline = 1;
	if (!nonline) {
		linecount++;
		nostartline = 0;
		nonline = 1;
	}
	if (c == '\n') nonline = 0;
	return c;
}

unput(c) {
	/*
	 * "unread" c
	 */
	backupc = c;
}

skipcomment(flag) {
	/*
	 * Skip comment. If flag != 0, the comment is inside a fragment
	 * of C-code, so the newlines in it must be copied to enable the
	 * C-compiler to keep a correct line count
	 */
{'fraction_non_alphanumeric': 0.15243423635649783, 'fraction_numerical': 0.007067137809187279, 'mean_word_length': 2.556369982547993, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6309486', 'n_tokens_mistral': 4343, 'n_tokens_neox': 3850, 'n_words': 1176}