<commit_before>
%{
/*
 * Copyright (c) 1999 <NAME> (<EMAIL>)
 *
 *    This source code is free software; you can redistribute it
 *    and/or modify it in source code form under the terms of the GNU
 *    General Public License as published by the Free Software
 *    Foundation; either version 2 of the License, or (at your option)
 *    any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
 */
#if !defined(WINNT)
#ident "$Id: lexor.lex,v 1.17 2000/03/29 04:36:42 steve Exp $"
#endif

# include  <stdio.h>
# include  <malloc.h>
# include  <string.h>
# include  <assert.h>

# include  "parse.h"
# include  "globals.h"

static void output_init();
#define YY_USER_INIT output_init()

static void def_match();
static void def_start();
static void def_finish();
static void def_undefine();
static void do_define();
static int  is_defined(const char*name);

static void include_filename();
static void do_include();
static int yywrap();


struct include_stack_t {
      char* path;

        /* If the current input is the the file, this member is set. */
      FILE*file;

        /* If we are reparsing a macro expansion, file is 0 and this
	   member points to the string is progress */
      const char*str;

      unsigned lineno;
      YY_BUFFER_STATE yybs;

      struct include_stack_t*next;
};

static struct include_stack_t*file_queue = 0;
static struct include_stack_t*istack  = 0;
static struct include_stack_t*standby = 0;

#define YY_INPUT(buf,result,max_size) do { \
    if (istack->file) { \
        size_t rc = fread(buf, 1, max_size, istack->file); \
        if (rc == 0) result = YY_NULL; \
        else result = rc; \
    } else { \
        if (*istack->str == 0) result = YY_NULL; \
        else { buf[0] = *istack->str++; result = 1; } \
    } \
} while(0)

static int comment_enter = 0;
%}

%option stack

%x PPINCLUDE
%x PPDEFINE
%x CCOMMENT
%x CSTRING

%x IFDEF_FALSE
%s IFDEF_TRUE
%x IFDEF_SUPR

W [ \t\b\f]+

%%

"//".* { ECHO; }

  /* detect multiline, c-style comments, passing them directly to the
     output. This is necessary to allow for ignoring directives that
     are included within the comments. */

"/*" { comment_enter = YY_START; BEGIN(CCOMMENT); ECHO; }
<CCOMMENT>.    { ECHO; }
<CCOMMENT>\n   { istack->lineno += 1; ECHO; }
<CCOMMENT>"*/" { BEGIN(comment_enter); ECHO; }

  /* Strings do not contain macros or preprocessor directives. */
\"            { comment_enter = YY_START; BEGIN(CSTRING); ECHO; }
<CSTRING>\\\" { yymore(); }
<CSTRING>\n   { yymore(); }
<CSTRING>\"   { BEGIN(comment_enter);  ECHO; }
<CSTRING>.    { yymore(); }

  /* This set of patterns matches the include directive and the name
     that follows it. when the directive ends, the do_include function
     performs the include operation. */

^{W}?`include { BEGIN(PPINCLUDE); }

<PPINCLUDE>\"[^\"]*\" { include_filename(); }

<PPINCLUDE>[ \t\b\f] { ; }

<PPINCLUDE>\n { istack->lineno += 1; BEGIN(0); do_include(); }
<PPINCLUDE><<EOF>> { istack->lineno += 1; BEGIN(0); do_include(); }


  /* Detect the define directive, and match the name. Match any
     white space that might be found, as well. After I get the
     directive and the name, go into PPDEFINE mode and prepare to
     collect the defined value. */

`define{W}[a-zA-Z][a-zA-Z0-9_]*{W}? { BEGIN(PPDEFINE); def_start(); }

<PPDEFINE>.* { do_define(); }

<PPDEFINE>\n {
      def_finish();
      istack->lineno += 1;
      fputc('\n', yyout);
      BEGIN(0);
  }

<PPDEFINE><<EOF>> {
      def_finish();
      istack->lineno += 1;
      fputc('\n', yyout);
      BEGIN(0);
  }

`undef{W}[a-zA-Z][a-zA-Z0-9_]*{W}?.* { def_undefine(); }


  /* Detect conditional compilation directives, and parse them. If I
     find the name defined, switch to the IFDEF_TRUE state and stay
     there until I get an `else or `endif. Otherwise, switch to the
     IFDEF_FALSE state and start tossing data.

     Handle suppressed `ifdef with an additional suppress start
     condition that stacks on top of the IFDEF_FALSE so that output is
     not accidentally turned on within nested ifdefs. */

^{W}?`ifdef{W}[a-zA-Z][a-zA-Z0-9_]*.* {
      char*name = strchr(yytext, '`');
      assert(name);
      name += 6;
      name += strspn(name, " \t\b\f");
      name[strcspn(name, " \t\b\f")] = 0;

      if (is_defined(name)) {
	    yy_push_state(IFDEF_TRUE);
      } else {
	    yy_push_state(IFDEF_FALSE);
      }
  }

<IFDEF_FALSE,IFDEF_SUPR>^{W}?`ifdef{W}.* { yy_push_state(IFDEF_SUPR); }<commit_msg> Handling mixing of defines within ifdef/endif.<commit_after>
%{
/*
 * Copyright (c) 1999 <NAME> (<EMAIL>)
 *
 *    This source code is free software; you can redistribute it
 *    and/or modify it in source code form under the terms of the GNU
 *    General Public License as published by the Free Software
 *    Foundation; either version 2 of the License, or (at your option)
 *    any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
 */
#if !defined(WINNT)
#ident "$Id: lexor.lex,v 1.18 2000/04/26 01:35:26 steve Exp $"
#endif

# include  <stdio.h>
# include  <malloc.h>
# include  <string.h>
# include  <assert.h>

# include  "parse.h"
# include  "globals.h"

static void output_init();
#define YY_USER_INIT output_init()

static void def_match();
static void def_start();
static void def_finish();
static void def_undefine();
static void do_define();
static int  is_defined(const char*name);

static void include_filename();
static void do_include();
static int yywrap();


struct include_stack_t {
      char* path;

        /* If the current input is the the file, this member is set. */
      FILE*file;

        /* If we are reparsing a macro expansion, file is 0 and this
	   member points to the string is progress */
      const char*str;

      unsigned lineno;
      YY_BUFFER_STATE yybs;

      struct include_stack_t*next;
};

static struct include_stack_t*file_queue = 0;
static struct include_stack_t*istack  = 0;
static struct include_stack_t*standby = 0;

#define YY_INPUT(buf,result,max_size) do { \
    if (istack->file) { \
        size_t rc = fread(buf, 1, max_size, istack->file); \
        if (rc == 0) result = YY_NULL; \
        else result = rc; \
    } else { \
        if (*istack->str == 0) result = YY_NULL; \
        else { buf[0] = *istack->str++; result = 1; } \
    } \
} while(0)

static int comment_enter = 0;
%}

%option stack

%x PPINCLUDE
%x PPDEFINE
%x CCOMMENT
%x CSTRING

%x IFDEF_FALSE
%s IFDEF_TRUE
%x IFDEF_SUPR

W [ \t\b\f]+

%%

"//".* { ECHO; }

  /* detect multiline, c-style comments, passing them directly to the
     output. This is necessary to allow for ignoring directives that
     are included within the comments. */

"/*" { comment_enter = YY_START; BEGIN(CCOMMENT); ECHO; }
<CCOMMENT>.    { ECHO; }
<CCOMMENT>\n   { istack->lineno += 1; ECHO; }
<CCOMMENT>"*/" { BEGIN(comment_enter); ECHO; }

  /* Strings do not contain macros or preprocessor directives. */
\"            { comment_enter = YY_START; BEGIN(CSTRING); ECHO; }
<CSTRING>\\\" { yymore(); }
<CSTRING>\n   { yymore(); }
<CSTRING>\"   { BEGIN(comment_enter);  ECHO; }
<CSTRING>.    { yymore(); }

  /* This set of patterns matches the include directive and the name
     that follows it. when the directive ends, the do_include function
     performs the include operation. */

^{W}?`include { yy_push_state(PPINCLUDE); }

<PPINCLUDE>\"[^\"]*\" { include_filename(); }

<PPINCLUDE>[ \t\b\f] { ; }

  /* Catch single-line comments that share the line with an include
     directive. And while I'm at it, I might as well preserve the
     comment in the output stream. */

<PPINCLUDE>"//".* { ECHO; }

  /* These finish the include directive (EOF or EOL) so I revert the
     lexor state and execute the inclusion. */

<PPINCLUDE>\n { istack->lineno += 1; yy_pop_state(); do_include(); }
<PPINCLUDE><<EOF>> { istack->lineno += 1; yy_pop_state(); do_include(); }


  /* Detect the define directive, and match the name. Match any
     white space that might be found, as well. After I get the
     directive and the name, go into PPDEFINE mode and prepare to
     collect the defined value. */

`define{W}[a-zA-Z][a-zA-Z0-9_]*{W}? { yy_push_state(PPDEFINE); def_start(); }

<PPDEFINE>.* { do_define(); }

<PPDEFINE>\n {
      def_finish();
      istack->lineno += 1;
      fputc('\n', yyout);
      yy_pop_state();
  }

<PPDEFINE><<EOF>> {
      def_finish();
      istack->lineno += 1;
      fputc('\n', yyout);
      yy_pop_state();
  }

`undef{W}[a-zA-Z][a-zA-Z0-9_]*{W}?.* { def_undefine(); }


  /* Detect conditional compilation directives, and parse them. If I
     find the name defined, switch to the IFDEF_TRUE state and stay
     there until I get an `else or `endif. Otherwise, switch to the
     IFDEF_FALSE state and start tossing data.

     Handle suppressed `ifdef with an additional suppress start
     condition that stacks on top of the IFDEF_FALSE so that output is
     not accidentally turned on within nested ifdefs. */

^{W}?`ifdef{W}[a-zA-Z][a-zA-Z0-9_]*.* {
      char*name = strchr(yytext, '`');
      assert(name);
      name += 6;
      name += strspn(name, " \t\b\f");
      name[strcspn(name, " \t\b\f")] = 0;

      if (is_defined(name)) {
	    yy_push_state(IFDEF_TRUE);
      } else {
	    yy_push_state(IFDEF_FALSE);
      }
  }

<IFDEF_FALSE,IFDEF_SUPR>^{W}?`ifdef{W}.* { yy_push_state(IFDEF_SUPR); }
{'fraction_non_alphanumeric': 0.1314218612388688, 'fraction_numerical': 0.012134259712300616, 'mean_word_length': 3.060389352403655, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '456239', 'n_tokens_mistral': 3635, 'n_tokens_neox': 3413, 'n_words': 1117}