<commit_before>/*
 * libmondai -- MONTSUQI data access library
 * Copyright (C) 2002-2003 Ogochan & JMA (Japan Medical Association).
 * Copyright (C) 2004-2008 Ogochan
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#ifndef	_INC_DD_LEX_H
#define	_INC_DD_LEX_H
#include	<glib.h>

#define	YYBASE			256
#define	T_EOF			(YYBASE +1)
#define	T_SYMBOL		(YYBASE +2)
#define	T_SCONST		(YYBASE +3)
#define	T_ICONST		(YYBASE +4)
#define	T_NCONST		(YYBASE +5)
#define	T_RCONST		(YYBASE +6)

#define	T_LT			'<'
#define	T_LE			(YYBASE +10)
#define	T_EQ			'='
#define	T_GE			(YYBASE +12)
#define	T_GT			'>'
#define	T_NE			(YYBASE +14)

#define	T_YYBASE		(YYBASE +20)

typedef	struct	INCFILE_S	{
	struct	INCFILE_S	*next;
	size_t				pos;
	size_t				size;
	int					cLine;
	FILE				*fp;
	char				*fn;
	char				*body;
}	INCFILE;

typedef	struct	_CURFILE_S {
	struct	_CURFILE_S	*next;
	GHashTable			*Reserved;
	INCFILE	*ftop;
	char	*body;
	int		back;
	size_t	pos;
	size_t	size;
	FILE	*fp;
	char	*fn;
	char	*path;
	int		cLine;
	Bool	fError;
	char	*Symbol;
	int		Int;
	int		Token;
	char	*ValueName;
}	CURFILE;

typedef	struct	{
	char	*str;
	int		token;
}	TokenTable;

#undef	GLOBAL
#ifdef	_LEX
#define	GLOBAL	/*	*/
#else
#define	GLOBAL	extern
#endif
GLOBAL	Bool	fLexVerbose;
#undef	GLOBAL

extern	CURFILE		*PushLexInfo(CURFILE *in, char *name, char *path, GHashTable *res);
extern	CURFILE		*PushLexInfoMem(CURFILE *in, char *mem, char *path, GHashTable *res);
extern	CURFILE		*PushLexInfoStream(CURFILE *in, FILE *fp, char *path, GHashTable *res);

extern	void			DropLexInfo(CURFILE **in);
extern	void			LexInit(void);
extern	GHashTable		*MakeReservedTable(TokenTable *table);
extern	void			SetReserved(CURFILE *in, GHashTable *res);

#define	LEX_GET_SYMBOL	1
#define	LEX_GET_NAME	2
#define	LEX_GET_STRING	3

extern	int				Lex(CURFILE *in, int type);
#define	GetSymbol		(ComToken = Lex(in,LEX_GET_SYMBOL))
#define	GetName			(ComToken = Lex(in,LEX_GET_NAME))
#define	GetString		(ComToken = Lex(in,LEX_GET_STRING))
#define	ComToken		(in->Token)
#define	ComInt			(in->Int)
#define	ComSymbol		(in->Symbol)

#define	ParError(msg)				{			\
	in->fError=TRUE;							\
	printf("%s:%d:%s\n",in->fn,in->cLine,msg);	\
	GetSymbol;									\
}
#define	ParErrorPrintf(...)	{					\
	in->fError=TRUE;							\
	printf("%s:%d:",in->fn,in->cLine);			\<commit_msg>The output is changed to stderr.<commit_after>/*
 * libmondai -- MONTSUQI data access library
 * Copyright (C) 2002-2003 Ogochan & JMA (Japan Medical Association).
 * Copyright (C) 2004-2008 Ogochan
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#ifndef	_INC_DD_LEX_H
#define	_INC_DD_LEX_H
#include	<glib.h>

#define	YYBASE			256
#define	T_EOF			(YYBASE +1)
#define	T_SYMBOL		(YYBASE +2)
#define	T_SCONST		(YYBASE +3)
#define	T_ICONST		(YYBASE +4)
#define	T_NCONST		(YYBASE +5)
#define	T_RCONST		(YYBASE +6)

#define	T_LT			'<'
#define	T_LE			(YYBASE +10)
#define	T_EQ			'='
#define	T_GE			(YYBASE +12)
#define	T_GT			'>'
#define	T_NE			(YYBASE +14)

#define	T_YYBASE		(YYBASE +20)

typedef	struct	INCFILE_S	{
	struct	INCFILE_S	*next;
	size_t				pos;
	size_t				size;
	int					cLine;
	FILE				*fp;
	char				*fn;
	char				*body;
}	INCFILE;

typedef	struct	_CURFILE_S {
	struct	_CURFILE_S	*next;
	GHashTable			*Reserved;
	INCFILE	*ftop;
	char	*body;
	int		back;
	size_t	pos;
	size_t	size;
	FILE	*fp;
	char	*fn;
	char	*path;
	int		cLine;
	Bool	fError;
	char	*Symbol;
	int		Int;
	int		Token;
	char	*ValueName;
}	CURFILE;

typedef	struct	{
	char	*str;
	int		token;
}	TokenTable;

#undef	GLOBAL
#ifdef	_LEX
#define	GLOBAL	/*	*/
#else
#define	GLOBAL	extern
#endif
GLOBAL	Bool	fLexVerbose;
#undef	GLOBAL

extern	CURFILE		*PushLexInfo(CURFILE *in, char *name, char *path, GHashTable *res);
extern	CURFILE		*PushLexInfoMem(CURFILE *in, char *mem, char *path, GHashTable *res);
extern	CURFILE		*PushLexInfoStream(CURFILE *in, FILE *fp, char *path, GHashTable *res);

extern	void			DropLexInfo(CURFILE **in);
extern	void			LexInit(void);
extern	GHashTable		*MakeReservedTable(TokenTable *table);
extern	void			SetReserved(CURFILE *in, GHashTable *res);

#define	LEX_GET_SYMBOL	1
#define	LEX_GET_NAME	2
#define	LEX_GET_STRING	3

extern	int				Lex(CURFILE *in, int type);
#define	GetSymbol		(ComToken = Lex(in,LEX_GET_SYMBOL))
#define	GetName			(ComToken = Lex(in,LEX_GET_NAME))
#define	GetString		(ComToken = Lex(in,LEX_GET_STRING))
#define	ComToken		(in->Token)
#define	ComInt			(in->Int)
#define	ComSymbol		(in->Symbol)

#define	ParError(msg)				{			\
	in->fError=TRUE;							\
	fprintf(stderr,"%s:%d:%s\n",in->fn,in->cLine,msg);	\
	GetSymbol;									\
}
#define	ParErrorPrintf(...)	{					\
	in->fError=TRUE;							\
	fprintf(stderr,"%s:%d:",in->fn,in->cLine);			\
{'fraction_non_alphanumeric': 0.1107454336021063, 'fraction_numerical': 0.016784597663320716, 'mean_word_length': 3.9616326530612245, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1134309', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2419, 'n_words': 725}