<commit_before>%{

 /*
    csound_prs.lex:

    Copyright (C) 2011, 2016
    <NAME>

    This file is part of Csound.

    The Csound Library is free software; you can redistribute it
    and/or modify it under the terms of the Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    Csound is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with Csound; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
    02110-1301 USA
*/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include "csoundCore.h"
#include "corfile.h"
#define YY_DECL int yylex (CSOUND *csound, yyscan_t yyscanner)
static void comment(yyscan_t);
static void do_comment(yyscan_t);
static void do_include(CSOUND *, int, yyscan_t);
static void do_new_include(CSOUND *, yyscan_t);
extern int isDir(char *);
static void do_macro_arg(CSOUND *, char *, yyscan_t);
static void do_macro(CSOUND *, char *, yyscan_t);
static void do_umacro(CSOUND *, char *, yyscan_t);
static void do_ifdef(CSOUND *, char *, yyscan_t);
static void do_ifdef_skip_code(CSOUND *, yyscan_t);
//static void print_csound_prsdata(CSOUND *,char *,yyscan_t);
static void csound_prs_line(CORFIL*, yyscan_t);
static void delete_macros(CSOUND*, yyscan_t);
#define MACDEBUG 1

#define S_INC (10)
static inline int isNameChar(int cc, int pos)
{
    unsigned char c = ((unsigned char) cc);
    return (isalpha(c) || (pos && (c == '_' || isdigit(c))));
}

#include "score_param.h"

//static void trace_alt_stack(CSOUND*, PRS_PARM*, int);

#define YY_EXTRA_TYPE  PRS_PARM *
#define PARM    yyget_extra(yyscanner)

#define YY_USER_INIT {csound_prs_scan_string(csound->scorestr->body, yyscanner); \
    csound_prsset_lineno(csound->scoLineOffset, yyscanner);             \
    /*yyg->yy_flex_debug_r=1;*/ PARM->macro_stack_size = 0;             \
    PARM->alt_stack = NULL; PARM->macro_stack_ptr = 0;                  \
    PARM->path = ".";                                                   \
    PARM->cf = csound->expanded_sco;                                    \
  }
static MACRO *find_definition(MACRO *, char *);
%}
%option reentrant
%option noyywrap
%option prefix="csound_prs"
%option outfile="Engine/csound_prslex.c"
%option stdout

NEWLINE         (\n|\r\n?)
STSTR           \"
ESCAPE          \\.
IDENT           [a-zA-Z_][a-zA-Z0-9_]*
MACRONAME       "$"`?[a-zA-Z_][a-zA-Z0-9_`]*
MACRONAMED      "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\.
MACRONAMEA      "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\(
MACRONAMEDA     "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\.\(
MACROB          [a-zA-Z_][a-zA-Z0-9_]*\(
MACRO           [a-zA-Z_][a-zA-Z0-9_]*
NUMBER [0-9]+\.?[0-9]*([eE][-+]?[0-9]+)?|\.[0-9]+([eE][-+]?[0-9]+)?|0[xX][0-9a-fA-F]+

STCOM           \/\*
INCLUDE         "#include"
INCLUDESTR      "#includestr"
DEFINE          #[ \t]*define
UNDEF           "#undef"
IFDEF           #ifn?def
ELSE            #else[ \t]*(;.*)?$
END             #end(if)?[ \t]*(;.*)?(\n|\r\n?)
LOOP            #loop
EXIT            #exit
CONT            \\[ \t]*(;.*)?(\n|\r\n?)
SEND            ^[ \t]*[es]
ROP             ^[ \t]*r

NP              np[0-9]+
NM              [nm][ \t]+

%X incl
%x macro
%x umacro
%x ifdef
%x lname

%%

{CONT}          {
#ifdef SCORE_PARSER
                  char bb[80];
#endif
                  csound_prsset_lineno(1+csound_prsget_lineno(yyscanner),
                                       yyscanner);
#ifdef SCORE_PARSER
                  if (PARM->isString==0) {
                    sprintf(bb, "#sline %d ", csound_prsget_lineno(yyscanner));
                    corfile_puts(csound, bb, PARM->cf);
                  }
#endif
                }
{NEWLINE}       {
                  corfile_putc(csound, '\n', PARM->cf);
                  csound_prsset_lineno(1+csound_prsget_lineno(yyscanner),
                                       yyscanner);
                  csound_prs_line(PARM->cf, yyscanner);
                }
"//"            {
                  if (PARM->isString != 1) {
                    comment(yyscanner);
                    corfile_putc(csound, '\n', PARM->cf);
                    csound_prs_line(PARM->cf, yyscanner);
                  }
                  else {
                    corfile_puts(csound, yytext, PARM->cf);
                  }
                }<commit_msg>Remove unised #sline in score lexer<commit_after>%{

 /*
    csound_prs.lex:

    Copyright (C) 2011, 2016
    <NAME>

    This file is part of Csound.

    The Csound Library is free software; you can redistribute it
    and/or modify it under the terms of the Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    Csound is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with Csound; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
    02110-1301 USA
*/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include "csoundCore.h"
#include "corfile.h"
#define YY_DECL int yylex (CSOUND *csound, yyscan_t yyscanner)
static void comment(yyscan_t);
static void do_comment(yyscan_t);
static void do_include(CSOUND *, int, yyscan_t);
static void do_new_include(CSOUND *, yyscan_t);
extern int isDir(char *);
static void do_macro_arg(CSOUND *, char *, yyscan_t);
static void do_macro(CSOUND *, char *, yyscan_t);
static void do_umacro(CSOUND *, char *, yyscan_t);
static void do_ifdef(CSOUND *, char *, yyscan_t);
static void do_ifdef_skip_code(CSOUND *, yyscan_t);
//static void print_csound_prsdata(CSOUND *,char *,yyscan_t);
static void csound_prs_line(CORFIL*, yyscan_t);
static void delete_macros(CSOUND*, yyscan_t);
#define MACDEBUG 1

#define S_INC (10)
static inline int isNameChar(int cc, int pos)
{
    unsigned char c = ((unsigned char) cc);
    return (isalpha(c) || (pos && (c == '_' || isdigit(c))));
}

#include "score_param.h"

//static void trace_alt_stack(CSOUND*, PRS_PARM*, int);

#define YY_EXTRA_TYPE  PRS_PARM *
#define PARM    yyget_extra(yyscanner)

#define YY_USER_INIT {csound_prs_scan_string(csound->scorestr->body, yyscanner); \
    csound_prsset_lineno(csound->scoLineOffset, yyscanner);             \
    /*yyg->yy_flex_debug_r=1;*/ PARM->macro_stack_size = 0;             \
    PARM->alt_stack = NULL; PARM->macro_stack_ptr = 0;                  \
    PARM->path = ".";                                                   \
    PARM->cf = csound->expanded_sco;                                    \
  }
static MACRO *find_definition(MACRO *, char *);
%}
%option reentrant
%option noyywrap
%option prefix="csound_prs"
%option outfile="Engine/csound_prslex.c"
%option stdout

NEWLINE         (\n|\r\n?)
STSTR           \"
ESCAPE          \\.
IDENT           [a-zA-Z_][a-zA-Z0-9_]*
MACRONAME       "$"`?[a-zA-Z_][a-zA-Z0-9_`]*
MACRONAMED      "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\.
MACRONAMEA      "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\(
MACRONAMEDA     "$"`?[a-zA-Z_][a-zA-Z0-9_`]*\.\(
MACROB          [a-zA-Z_][a-zA-Z0-9_]*\(
MACRO           [a-zA-Z_][a-zA-Z0-9_]*
NUMBER [0-9]+\.?[0-9]*([eE][-+]?[0-9]+)?|\.[0-9]+([eE][-+]?[0-9]+)?|0[xX][0-9a-fA-F]+

STCOM           \/\*
INCLUDE         "#include"
INCLUDESTR      "#includestr"
DEFINE          #[ \t]*define
UNDEF           "#undef"
IFDEF           #ifn?def
ELSE            #else[ \t]*(;.*)?$
END             #end(if)?[ \t]*(;.*)?(\n|\r\n?)
LOOP            #loop
EXIT            #exit
CONT            \\[ \t]*(;.*)?(\n|\r\n?)
SEND            ^[ \t]*[es]
ROP             ^[ \t]*r

NP              np[0-9]+
NM              [nm][ \t]+

%X incl
%x macro
%x umacro
%x ifdef
%x lname

%%

{CONT}          {
#if 0
                  char bb[80];
#endif
                  csound_prsset_lineno(1+csound_prsget_lineno(yyscanner),
                                       yyscanner);
#if 0
                  if (PARM->isString==0) {
                    sprintf(bb, "#sline %d ", csound_prsget_lineno(yyscanner));
                    corfile_puts(csound, bb, PARM->cf);
                  }
#endif
                }
{NEWLINE}       {
                  corfile_putc(csound, '\n', PARM->cf);
                  csound_prsset_lineno(1+csound_prsget_lineno(yyscanner),
                                       yyscanner);
                  csound_prs_line(PARM->cf, yyscanner);
                }
"//"            {
                  if (PARM->isString != 1) {
                    comment(yyscanner);
                    corfile_putc(csound, '\n', PARM->cf);
                    csound_prs_line(PARM->cf, yyscanner);
                  }
                  else {
                    corfile_puts(csound, yytext, PARM->cf);
                  }
                }
{'fraction_non_alphanumeric': 0.14368191721132897, 'fraction_numerical': 0.013725490196078431, 'mean_word_length': 2.04208084824387, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5177039', 'n_tokens_mistral': 3433, 'n_tokens_neox': 3304, 'n_words': 729}