<commit_before>{
module Language.CSharp.Lexer
    ( L (..)
    , Token (..)
    , lexer
    ) where

import qualified Data.Text as T
import           Numeric
}

--%wrapper "posn-bytestring"

$any     = [.\n\r]
@newline = [\n\r] | \r\n
@comment = "/*" $any* "*/"
         | "//" .* @newline

@preprocessor = \# .* @newline

-- C# actually defines a letter to be any character (or escape sequence)
-- from the Unicode classes Lu, Ll, Lt, Lm, Lo or Nl. Identifiers must
-- start with a letter or an underscore, but can then also contain
-- characters from the classes Mn, Mc, Nd, Pc or Cf.
$ident_start = [a-zA-Z_\@]
$ident_part  = [a-zA-Z_0-9]

$digit     = [0-9]
$hex_digit = [0-9a-fA-F]
$sign      = [\+\-]

@int_suffix  = [uU][lL]? | [lL][uU]?
@real_suffix = [fFdDmM]
@exponent    = [eE] $sign? $digit+

@simple_escape  = \\ [0abfnrtv\'\"\\]
@hex_escape     = \\x $hex_digit{1,4}
@unicode_escape = \\u $hex_digit{4} | \\U $hex_digit{8}
@escapes        = @simple_escape | @hex_escape | @unicode_escape

@character          = [^\'\\] | @escapes
@string_character   = [^\"\\] | @escapes
@verbatim_character = $any # \" | \"\"

-- Not sure how to deal with these yet, so we'll just ignore them
--@bom = \xEF \xBB \xBF -- UTF-8
--     | \uFEFF         -- UTF-16BE
--     | \uFFFE         -- UTF-16LE

tokens :-

$white+       ;
@comment      ;
@preprocessor ;
--@bom          ;

-- Keywords
abstract   { constTok Tok_Abstract   }
as         { constTok Tok_As         }
base       { constTok Tok_Base       }
bool       { constTok Tok_Bool       }
break      { constTok Tok_Break      }
byte       { constTok Tok_Byte       }
case       { constTok Tok_Case       }
catch      { constTok Tok_Catch      }
char       { constTok Tok_Char       }
checked    { constTok Tok_Checked    }
class      { constTok Tok_Class      }
const      { constTok Tok_Const      }
continue   { constTok Tok_Continue   }
decimal    { constTok Tok_Decimal    }
default    { constTok Tok_Default    }
delegate   { constTok Tok_Delegate   }
do         { constTok Tok_Do         }
double     { constTok Tok_Double     }<commit_msg>Ignore \NUL, maybe not a good idea? we'll see<commit_after>{
module Language.CSharp.Lexer
    ( L (..)
    , Token (..)
    , lexer
    ) where

import qualified Data.Text as T
import           Numeric
}

--%wrapper "posn-bytestring"

$any     = [.\n\r]
@newline = [\n\r] | \r\n
@comment = "/*" $any* "*/"
         | "//" .* @newline

@preprocessor = \# .* @newline

-- C# actually defines a letter to be any character (or escape sequence)
-- from the Unicode classes Lu, Ll, Lt, Lm, Lo or Nl. Identifiers must
-- start with a letter or an underscore, but can then also contain
-- characters from the classes Mn, Mc, Nd, Pc or Cf.
$ident_start = [a-zA-Z_\@]
$ident_part  = [a-zA-Z_0-9]

$digit     = [0-9]
$hex_digit = [0-9a-fA-F]
$sign      = [\+\-]

@int_suffix  = [uU][lL]? | [lL][uU]?
@real_suffix = [fFdDmM]
@exponent    = [eE] $sign? $digit+

@simple_escape  = \\ [0abfnrtv\'\"\\]
@hex_escape     = \\x $hex_digit{1,4}
@unicode_escape = \\u $hex_digit{4} | \\U $hex_digit{8}
@escapes        = @simple_escape | @hex_escape | @unicode_escape

@character          = [^\'\\] | @escapes
@string_character   = [^\"\\] | @escapes
@verbatim_character = $any # \" | \"\"

-- Not sure how to deal with these yet, so we'll just ignore them
--@bom = \xEF \xBB \xBF -- UTF-8
--     | \uFEFF         -- UTF-16BE
--     | \uFFFE         -- UTF-16LE

tokens :-

$white+       ;
\0+           ;
@comment      ;
@preprocessor ;
--@bom          ;

-- Keywords
abstract   { constTok Tok_Abstract   }
as         { constTok Tok_As         }
base       { constTok Tok_Base       }
bool       { constTok Tok_Bool       }
break      { constTok Tok_Break      }
byte       { constTok Tok_Byte       }
case       { constTok Tok_Case       }
catch      { constTok Tok_Catch      }
char       { constTok Tok_Char       }
checked    { constTok Tok_Checked    }
class      { constTok Tok_Class      }
const      { constTok Tok_Const      }
continue   { constTok Tok_Continue   }
decimal    { constTok Tok_Decimal    }
default    { constTok Tok_Default    }
delegate   { constTok Tok_Delegate   }
do         { constTok Tok_Do         }
double     { constTok Tok_Double     }
{'fraction_non_alphanumeric': 0.14383239171374765, 'fraction_numerical': 0.007768361581920904, 'mean_word_length': 2.0156139105748756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6931082', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1526, 'n_words': 395}