<commit_before>{
module Tokenizer (Token(..), tokenize) where
}

%wrapper "basic"

$digit = 0-9
$hexdigit = [0-9a-fA-F]
$character = [a-zA-Z_]
$eol = [\n]

tokens :-
  $eol              ;
  $white+           ;
  "#".*             ;
  $digit+           { \s -> TokenInt (read s) }
  "0x"$hexdigit+    { \s -> TokenHexInt (read s) }
  in                { \s -> TokenIn }
  and               { \s -> TokenAnd }
  or                { \s -> TokenOr }
  "?"               { \s -> TokenQuestionMark }
  ":"               { \s -> TokenColon }
  "+"               { \s -> TokenPlus }
  "-"               { \s -> TokenMinus }
  "*"               { \s -> TokenMultiply }
  "**"              { \s -> TokenPower }
  "<<"              { \s -> TokenShiftLeft }
  ">>"              { \s -> TokenShiftRight }
  ">>>"             { \s -> TokenShiftRightArithmetic }
  "<"               { \s -> TokenLessThan }
  ">"               { \s -> TokenGreaterThan }
  "=="              { \s -> TokenEqual }
  "&"               { \s -> TokenBinaryAnd }
  "|"               { \s -> TokenBinaryOr }
  "^"               { \s -> TokenBinaryXor }
  "="               { \s -> TokenAssignment }
  "|="              { \s -> TokenAssignmentOr }
  "&="              { \s -> TokenAssignmentAnd }
  "+="              { \s -> TokenAssignmentPlus }
  "-="              { \s -> TokenAssignmentMinus }
  "*="              { \s -> TokenAssignmentMultiply }
  "$"               { \s -> TokenDollar }
  \(                { \s -> TokenLParen }
  \)                { \s -> TokenRParen }
  \[                { \s -> TokenLBracket }
  \]                { \s -> TokenRBracket }
  ","               { \s -> TokenComma }

  "load!"           { \s -> TokenLoad }
  "store!"          { \s -> TokenStore }
  "constants"       { \s -> TokenConstants }
  $character+       { \s -> TokenIdentifier s }

{

data Token = TokenIn
           | TokenAnd
           | TokenOr
           | TokenQuestionMark
           | TokenColon
           | TokenPlus
           | TokenMinus
           | TokenMultiply
           | TokenPower
           | TokenShiftLeft
           | TokenShiftRight
           | TokenShiftRightArithmetic
           | TokenLessThan
           | TokenGreaterThan
           | TokenEqual
           | TokenBinaryAnd
           | TokenBinaryOr
           | TokenBinaryXor<commit_msg>Allow numbers in var names<commit_after>{
module Tokenizer (Token(..), tokenize) where
}

%wrapper "basic"

$digit = 0-9
$hexdigit = [0-9a-fA-F]
$firstcharacter = [a-zA-Z_]
$character = [a-zA-Z0-9_]
$eol = [\n]

tokens :-
  $eol              ;
  $white+           ;
  "#".*             ;
  $digit+           { \s -> TokenInt (read s) }
  "0x"$hexdigit+    { \s -> TokenHexInt (read s) }
  in                { \s -> TokenIn }
  and               { \s -> TokenAnd }
  or                { \s -> TokenOr }
  "?"               { \s -> TokenQuestionMark }
  ":"               { \s -> TokenColon }
  "+"               { \s -> TokenPlus }
  "-"               { \s -> TokenMinus }
  "*"               { \s -> TokenMultiply }
  "**"              { \s -> TokenPower }
  "<<"              { \s -> TokenShiftLeft }
  ">>"              { \s -> TokenShiftRight }
  ">>>"             { \s -> TokenShiftRightArithmetic }
  "<"               { \s -> TokenLessThan }
  ">"               { \s -> TokenGreaterThan }
  "=="              { \s -> TokenEqual }
  "&"               { \s -> TokenBinaryAnd }
  "|"               { \s -> TokenBinaryOr }
  "^"               { \s -> TokenBinaryXor }
  "="               { \s -> TokenAssignment }
  "|="              { \s -> TokenAssignmentOr }
  "&="              { \s -> TokenAssignmentAnd }
  "+="              { \s -> TokenAssignmentPlus }
  "-="              { \s -> TokenAssignmentMinus }
  "*="              { \s -> TokenAssignmentMultiply }
  "$"               { \s -> TokenDollar }
  \(                { \s -> TokenLParen }
  \)                { \s -> TokenRParen }
  \[                { \s -> TokenLBracket }
  \]                { \s -> TokenRBracket }
  ","               { \s -> TokenComma }

  "load!"           { \s -> TokenLoad }
  "store!"          { \s -> TokenStore }
  "constants"       { \s -> TokenConstants }
  $firstcharacter$character* { \s -> TokenIdentifier s }

{

data Token = TokenIn
           | TokenAnd
           | TokenOr
           | TokenQuestionMark
           | TokenColon
           | TokenPlus
           | TokenMinus
           | TokenMultiply
           | TokenPower
           | TokenShiftLeft
           | TokenShiftRight
           | TokenShiftRightArithmetic
           | TokenLessThan
           | TokenGreaterThan
           | TokenEqual
           | TokenBinaryAnd
           | TokenBinaryOr
           | TokenBinaryXor
{'fraction_non_alphanumeric': 0.15251006142766363, 'fraction_numerical': 0.0025418343571277273, 'mean_word_length': 1.0629095674967235, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4069558', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1418, 'n_words': 255}