<commit_before>
/* -*- C++ -*- */

/*
  (c) COPYRIGHT URI/MIT 1994-1996
  Please read the full copyright statement in the file COPYRIGH.  

  Authors:
	jhrg,jimg       <NAME> (<EMAIL>)
*/

/*
  Scanner for constraint expressions. The scanner returns tokens for each of
  the relational and selection operators. It requires GNU flex version 2.5.2
  or newer.

  The scanner is not reentrant, but can share a name space with other
  scanners. 

   Note:
   1) The `defines' file expr.tab.h is built using `bison -d'.
   2) Define YY_DECL such that the scanner is called `exprlex'.
   3) When bison builds the expr.tab.h file, it uses `expr' instead of `yy'
   for variable name prefixes (e.g., yylval --> exprlval).

  jhrg 9/5/95
*/

/* 
 * $Log: expr.lex,v $
 * Revision 1.16  1998/03/26 00:26:23  jimg
 * Added % to the set of characters that can start and ID
 *
 * Revision 1.15  1997/08/11 18:19:36  jimg
 * Fixed comment leaders for new CVS version
 *
 * Revision 1.14  1997/02/24 18:18:23  jimg
 * Removed `rule' for "." since it cannot be matched.
 *
 * Revision 1.13  1996/11/13 19:23:15  jimg
 * Fixed debugging.
 *
 * Revision 1.12  1996/10/08 17:10:52  jimg
 * Added % to the set of characters allowable in identifier names
 *
 * Revision 1.11  1996/08/26 21:13:17  jimg
 * Changes for version 2.07
 *
 * Revision 1.10  1996/08/13 18:56:24  jimg
 * Added __unused__ to definition of char rcsid[].
 *
 * Revision 1.9  1996/05/31 23:31:03  jimg
 * Updated copyright notice.
 *
 * Revision 1.8  1996/05/14 15:39:01  jimg
 * These changes have already been checked in once before. However, I
 * corrupted the source repository and restored it from a 5/9/96 backup
 * tape. The previous version's log entry should cover the changes.
 *
 * Revision 1.7  1996/04/05 00:22:19  jimg
 * Compiled with g++ -Wall and fixed various warnings.
 *
 * Revision 1.6  1996/03/02 01:19:04  jimg
 * Fixed comments.
 * Fixed a bug in store_str(); leading and trailing double quotes are now
 * stripped from strings.
 *
 * Revision 1.5  1996/02/01 17:43:16  jimg
 * Added support for lists as operands in constraint expressions.
 *
 * Revision 1.4  1995/12/09  01:07:39  jimg
 * Added changes so that relational operators will work properly for all the
 * datatypes (including Sequences). The relational ops are evaluated in
 * DDS::eval_constraint() after being parsed by DDS::parse_constraint().
 *
 * Revision 1.3  1995/12/06  18:57:37  jimg
 * Because the %union{} changed, the return types of some of the rules also
 * changed.
 * Returns integer codes for relops.
 * Returns a tagged union for most other values.
 *
 * Revision 1.2  1995/10/23  23:11:31  jimg
 * Fixed scanner to use the new definition of YYSTYPE.
 *
 * Revision 1.1  1995/10/13  03:03:17  jimg
 * Scanner. Incorporates Glenn's suggestions.
 *
 */

%{

#include "config_dap.h"

static char rcsid[] __unused__ = {"$Id: expr.lex,v 1.16 1998/03/26 00:26:23 jimg Exp $"};

#include <string.h>
#include <assert.h>

#include <String.h>
#include <SLList.h>

#define YY_DECL int exprlex YY_PROTO(( void ))

#include "Error.h"
#include "parser.h"
#include "expr.h"
#include "expr.tab.h"

void store_int32();
void store_float64();
void store_id();
void store_str();
void store_op(int op);

%}

%x quote
    
ID		[a-zA-Z_%][a-zA-Z0-9_/%]*
FIELD           {ID}\.{ID}(\.{ID})*
INT		[-+]?[0-9]+

MANTISA		([0-9]+\.?[0-9]*)|([0-9]*\.?[0-9]+)
EXPONENT	(E|e)[-+]?[0-9]+

FLOAT		[-+]?{MANTISA}{EXPONENT}?

STR		[-+a-zA-Z0-9_/.%]+

EQUAL		=
NOT_EQUAL	!=
GREATER		>
GREATER_EQL	>=
LESS		<
LESS_EQL	<=<commit_msg>Now includes RValue.h. Needed because expr.tab.h needs it.<commit_after>
/* -*- C++ -*- */

/*
  (c) COPYRIGHT URI/MIT 1994-1996
  Please read the full copyright statement in the file COPYRIGH.  

  Authors:
	jhrg,jimg       <NAME> (<EMAIL>)
*/

/*
  Scanner for constraint expressions. The scanner returns tokens for each of
  the relational and selection operators. It requires GNU flex version 2.5.2
  or newer.

  The scanner is not reentrant, but can share a name space with other
  scanners. 

   Note:
   1) The `defines' file expr.tab.h is built using `bison -d'.
   2) Define YY_DECL such that the scanner is called `exprlex'.
   3) When bison builds the expr.tab.h file, it uses `expr' instead of `yy'
   for variable name prefixes (e.g., yylval --> exprlval).

  jhrg 9/5/95
*/

/* 
 * $Log: expr.lex,v $
 * Revision 1.17  1998/10/21 16:45:50  jimg
 * Now includes RValue.h. Needed because expr.tab.h needs it.
 *
 * Revision 1.16  1998/03/26 00:26:23  jimg
 * Added % to the set of characters that can start and ID
 *
 * Revision 1.15  1997/08/11 18:19:36  jimg
 * Fixed comment leaders for new CVS version
 *
 * Revision 1.14  1997/02/24 18:18:23  jimg
 * Removed `rule' for "." since it cannot be matched.
 *
 * Revision 1.13  1996/11/13 19:23:15  jimg
 * Fixed debugging.
 *
 * Revision 1.12  1996/10/08 17:10:52  jimg
 * Added % to the set of characters allowable in identifier names
 *
 * Revision 1.11  1996/08/26 21:13:17  jimg
 * Changes for version 2.07
 *
 * Revision 1.10  1996/08/13 18:56:24  jimg
 * Added __unused__ to definition of char rcsid[].
 *
 * Revision 1.9  1996/05/31 23:31:03  jimg
 * Updated copyright notice.
 *
 * Revision 1.8  1996/05/14 15:39:01  jimg
 * These changes have already been checked in once before. However, I
 * corrupted the source repository and restored it from a 5/9/96 backup
 * tape. The previous version's log entry should cover the changes.
 *
 * Revision 1.7  1996/04/05 00:22:19  jimg
 * Compiled with g++ -Wall and fixed various warnings.
 *
 * Revision 1.6  1996/03/02 01:19:04  jimg
 * Fixed comments.
 * Fixed a bug in store_str(); leading and trailing double quotes are now
 * stripped from strings.
 *
 * Revision 1.5  1996/02/01 17:43:16  jimg
 * Added support for lists as operands in constraint expressions.
 *
 * Revision 1.4  1995/12/09  01:07:39  jimg
 * Added changes so that relational operators will work properly for all the
 * datatypes (including Sequences). The relational ops are evaluated in
 * DDS::eval_constraint() after being parsed by DDS::parse_constraint().
 *
 * Revision 1.3  1995/12/06  18:57:37  jimg
 * Because the %union{} changed, the return types of some of the rules also
 * changed.
 * Returns integer codes for relops.
 * Returns a tagged union for most other values.
 *
 * Revision 1.2  1995/10/23  23:11:31  jimg
 * Fixed scanner to use the new definition of YYSTYPE.
 *
 * Revision 1.1  1995/10/13  03:03:17  jimg
 * Scanner. Incorporates Glenn's suggestions.
 *
 */

%{

#include "config_dap.h"

static char rcsid[] __unused__ = {"$Id: expr.lex,v 1.17 1998/10/21 16:45:50 jimg Exp $"};

#include <string.h>
#include <assert.h>

#include <String.h>
#include <SLList.h>

#define YY_DECL int exprlex YY_PROTO(( void ))

#include "Error.h"
#include "parser.h"
#include "expr.h"
#include "RValue.h"
#include "expr.tab.h"

void store_int32();
void store_float64();
void store_id();
void store_str();
void store_op(int op);

%}

%x quote
    
ID		[a-zA-Z_%][a-zA-Z0-9_/%]*
FIELD           {ID}\.{ID}(\.{ID})*
INT		[-+]?[0-9]+

MANTISA		([0-9]+\.?[0-9]*)|([0-9]*\.?[0-9]+)
EXPONENT	(E|e)[-+]?[0-9]+

FLOAT		[-+]?{MANTISA}{EXPONENT}?

STR		[-+a-zA-Z0-9_/.%]+

EQUAL		=
NOT_EQUAL	!=
GREATER		>
GREATER_EQL	>=
LESS		<
LESS_EQL	<=
{'fraction_non_alphanumeric': 0.1297059631766969, 'fraction_numerical': 0.0916460566089585, 'mean_word_length': 3.1570531125071386, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7158391', 'n_tokens_mistral': 3160, 'n_tokens_neox': 2686, 'n_words': 896}