<commit_before>%{
#include <string.h>
#include "whistlepig.h"
#include "query-parser.h"
#include "query-parser.tab.h"

#define YY_EXTRA_TYPE query_parse_context*
#define YY_USER_ACTION yylloc->first_line = yylineno;

#define YY_INPUT(buf,result,max_size) \
{                                     \
  if(yyextra->input[0] == 0)          \
     result = YY_NULL;                \
  else {                              \
     buf[0] = yyextra->input[0];      \
     yyextra->input++;                \
     result = 1;                      \
  }                                   \
}

%}

%option 8bit reentrant fast noyywrap yylineno
%option outfile="query-parser.lex.c" header-file="query-parser.lex.h"
%option prefix="query_parser_"
%option bison-bridge bison-locations

FIRSTWORDCHAR [[:alnum:]]
INNERWORDCHAR [[:alnum:]'_#@\-\.\/]
LASTWORDCHAR  [[:alnum:]_#@-]

%%

OR return OR;

{FIRSTWORDCHAR}{INNERWORDCHAR}*{LASTWORDCHAR} {
  yylval->string = strdup(yytext);
  return WORD;
}

{FIRSTWORDCHAR}{LASTWORDCHAR}? {
  yylval->string = strdup(yytext);
  return WORD;
}

[\n[:blank:]] { } ; // nothing

. return yytext[0];

%%
<commit_msg>fix query lexer to be much more permissive<commit_after>%{
#include <string.h>
#include "whistlepig.h"
#include "query-parser.h"
#include "query-parser.tab.h"

#define YY_EXTRA_TYPE query_parse_context*
#define YY_USER_ACTION yylloc->first_line = yylineno;

#define YY_INPUT(buf,result,max_size) \
{                                     \
  if(yyextra->input[0] == 0)          \
     result = YY_NULL;                \
  else {                              \
     buf[0] = yyextra->input[0];      \
     yyextra->input++;                \
     result = 1;                      \
  }                                   \
}

%}

%option 8bit reentrant fast noyywrap yylineno
%option outfile="query-parser.lex.c" header-file="query-parser.lex.h"
%option prefix="query_parser_"
%option bison-bridge bison-locations

/* for the first char, everything is allowed except ()"-~ */
FIRSTWORDCHAR [[:alnum:]+=!@#$%\^&\*_|'/\?\.\,<>`]

/* inside a word, everything is allowed except ()" */
INNERWORDCHAR [[:alnum:]+=!@#$%\^&\*_|'/\?\.\,<>`\-~]

%%

OR return OR;

{FIRSTWORDCHAR}{INNERWORDCHAR}* {
  yylval->string = strdup(yytext);
  return WORD;
}

[\n[:blank:]] ; // nothing

. return yytext[0];

%%

{'fraction_non_alphanumeric': 0.16809974204643163, 'fraction_numerical': 0.0060189165950128975, 'mean_word_length': 2.209655172413793, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7102238', 'n_tokens_mistral': 864, 'n_tokens_neox': 836, 'n_words': 145}