<commit_before>%
% reia_scan: Leex scanner for the Reia language
% Copyright (C)2008-09 <NAME>
% 
% Redistribution is permitted under the MIT license.  See LICENSE for details.
%

Definitions.

Digit = [0-9]
UpperCase = [A-Z]
LowerCase = [a-z]
Whitespace = [\s]
DoubleQuoted = "(\\\^.|\\.|[^\"])*"
SingleQuoted = '(\\\^.|\\.|[^\'])*'
Regexp = \%r\/(\\\^.|\\.|[^/])*\/
Comment = #.*

Rules.

%% Numbers
-?{Digit}+\.{Digit}+ : build_float(TokenChars, TokenLine).
-?{Digit}+           : build_integer(TokenChars, TokenLine).

%% Strings
{DoubleQuoted} : build_string(TokenChars, TokenLine, TokenLen).
{SingleQuoted} : build_string(TokenChars, TokenLine, TokenLen).

%% Regular expressions
{Regexp} : build_regexp(TokenChars, TokenLine, TokenLen).

%% Atoms
\:({UpperCase}|{LowerCase}|_)({UpperCase}|{Digit}|{LowerCase}|_)* : build_atom(TokenChars, TokenLine, TokenLen).
\:{DoubleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).
\:{SingleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).

%% Module and Variable Names
{UpperCase}({UpperCase}|{LowerCase}|{Digit}|_)* : build_module_name(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)* : build_identifier(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)*[?!] : build_punctuated_identifier(TokenChars, TokenLine).

%% Ignored
{Comment} : skip_token.
{Whitespace}+ : skip_token.

%% Newline-spanning whitespace
({Comment}|{Whitespace})*(\n({Comment}|{Whitespace})*)+ : {token,{eol,TokenLine}}.

\(    : {token,{'(',TokenLine}}.
\)    : {token,{')',TokenLine}}.
\[    : {token,{'[',TokenLine}}.
\]    : {token,{']',TokenLine}}.
\{    : {token,{'{',TokenLine}}.
}     : {token,{'}',TokenLine}}.
<-    : {token,{'<-',TokenLine}}.
->    : {token,{'->',TokenLine}}.
<<    : {token,{'<<',TokenLine}}.
>>    : {token,{'>>',TokenLine}}.
\+    : {token,{'+',TokenLine}}.
-     : {token,{'-',TokenLine}}.
\*    : {token,{'*',TokenLine}}.
/     : {token,{'/',TokenLine}}.
\%    : {token,{'%',TokenLine}}.
\*\*  : {token,{'**',TokenLine}}.
\.    : {token,{'.',TokenLine}}.
,     : {token,{',',TokenLine}}.
:     : {token,{':',TokenLine}}.
::    : {token,{'::',TokenLine}}.
;     : {token,{'eol',TokenLine}}.
@     : {token,{'@',TokenLine}}.
=     : {token,{'=',TokenLine}}.
==    : {token,{'==',TokenLine}}.
===   : {token,{'===',TokenLine}}.
!=    : {token,{'!=',TokenLine}}.
<     : {token,{'<',TokenLine}}.
>     : {token,{'>',TokenLine}}.
<\[   : {token,{'<[',TokenLine}}.
\]>   : {token,{']>',TokenLine}}.
=>    : {token,{'=>',TokenLine}}.
<=    : {token,{'<=',TokenLine}}.
>=    : {token,{'>=',TokenLine}}.
\+=   : {token,{'+=',TokenLine}}.
-=    : {token,{'-=',TokenLine}}.
\*=   : {token,{'*=',TokenLine}}.
/=    : {token,{'/=',TokenLine}}.
\*\*= : {token,{'**=',TokenLine}}.
\.\.  : {token,{'..',TokenLine}}.
\|    : {token,{'|',TokenLine}}.
\|\|  : {token,{'or',TokenLine}}.
\|\|= : {token,{'||=',TokenLine}}.
&     : {token,{'&',TokenLine}}.
&&    : {token,{'and',TokenLine}}.
\^    : {token,{'^',TokenLine}}.
~     : {token,{'~',TokenLine}}.
\?    : {token,{'?',TokenLine}}.
!     : {token,{'!',TokenLine}}.
\$    : {token,{'$',TokenLine}}.
&=    : {token,{'&=',TokenLine}}.
\|=   : {token,{'|=',TokenLine}}.
\^=   : {token,{'^=',TokenLine}}.
<<=   : {token,{'<<=',TokenLine}}.
>>=   : {token,{'>>=',TokenLine}}.

Erlang code.  

-include("reia_nodes.hrl").

build_integer([$-|Chars], Line) ->
  {token, #integer{line = Line, value = -list_to_integer(Chars)}};
build_integer(Chars, Line) ->
  {token, #integer{line = Line, value =  list_to_integer(Chars)}}.

build_float([$-|Chars], Line) ->
  {token, #float{line = Line, value = -list_to_float(Chars)}};
build_float(Chars, Line) ->
  {token, #float{line = Line, value =  list_to_float(Chars)}}.
  
build_string(Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 2, Len - 2)), 
  {token, {string, Line, String}}.

build_regexp(Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 4, Len - 4)),
  {token, {regexp, Line, String}}.
  
unescape_string(String) -> unescape_string(String, []).

unescape_string([], Output) ->
  lists:reverse(Output);
unescape_string([$\\, Escaped | Rest], Output) ->
  Char = case Escaped of
    $\\ -> $\\;
    $/  -> $/; 
    $\" -> $\";
    $\' -> $\';
    $b  -> $\b;
    $d  -> $\d;
    $e  -> $\e;
    $f  -> $\f;
    $n  -> $\n;
    $r  -> $\r;<commit_msg>Remove all processing of unaries and the <- token in the lexer<commit_after>%
% reia_scan: Leex scanner for the Reia language
% Copyright (C)2008-09 <NAME>
% 
% Redistribution is permitted under the MIT license.  See LICENSE for details.
%

Definitions.

Digit = [0-9]
UpperCase = [A-Z]
LowerCase = [a-z]
Whitespace = [\s]
DoubleQuoted = "(\\\^.|\\.|[^\"])*"
SingleQuoted = '(\\\^.|\\.|[^\'])*'
Regexp = \%r\/(\\\^.|\\.|[^/])*\/
Comment = #.*

Rules.

%% Numbers
{Digit}+\.{Digit}+ : build_float(TokenChars, TokenLine).
{Digit}+           : build_integer(TokenChars, TokenLine).

%% Strings
{DoubleQuoted} : build_string(TokenChars, TokenLine, TokenLen).
{SingleQuoted} : build_string(TokenChars, TokenLine, TokenLen).

%% Regular expressions
{Regexp} : build_regexp(TokenChars, TokenLine, TokenLen).

%% Atoms
\:({UpperCase}|{LowerCase}|_)({UpperCase}|{Digit}|{LowerCase}|_)* : build_atom(TokenChars, TokenLine, TokenLen).
\:{DoubleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).
\:{SingleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).

%% Module and Variable Names
{UpperCase}({UpperCase}|{LowerCase}|{Digit}|_)* : build_module_name(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)* : build_identifier(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)*[?!] : build_punctuated_identifier(TokenChars, TokenLine).

%% Ignored
{Comment} : skip_token.
{Whitespace}+ : skip_token.

%% Newline-spanning whitespace
({Comment}|{Whitespace})*(\n({Comment}|{Whitespace})*)+ : {token,{eol,TokenLine}}.

\(    : {token,{'(',TokenLine}}.
\)    : {token,{')',TokenLine}}.
\[    : {token,{'[',TokenLine}}.
\]    : {token,{']',TokenLine}}.
\{    : {token,{'{',TokenLine}}.
}     : {token,{'}',TokenLine}}.
->    : {token,{'->',TokenLine}}.
<<    : {token,{'<<',TokenLine}}.
>>    : {token,{'>>',TokenLine}}.
\+    : {token,{'+',TokenLine}}.
-     : {token,{'-',TokenLine}}.
\*    : {token,{'*',TokenLine}}.
/     : {token,{'/',TokenLine}}.
\%    : {token,{'%',TokenLine}}.
\*\*  : {token,{'**',TokenLine}}.
\.    : {token,{'.',TokenLine}}.
,     : {token,{',',TokenLine}}.
:     : {token,{':',TokenLine}}.
::    : {token,{'::',TokenLine}}.
;     : {token,{'eol',TokenLine}}.
@     : {token,{'@',TokenLine}}.
=     : {token,{'=',TokenLine}}.
==    : {token,{'==',TokenLine}}.
===   : {token,{'===',TokenLine}}.
!=    : {token,{'!=',TokenLine}}.
<     : {token,{'<',TokenLine}}.
>     : {token,{'>',TokenLine}}.
<\[   : {token,{'<[',TokenLine}}.
\]>   : {token,{']>',TokenLine}}.
=>    : {token,{'=>',TokenLine}}.
<=    : {token,{'<=',TokenLine}}.
>=    : {token,{'>=',TokenLine}}.
\+=   : {token,{'+=',TokenLine}}.
-=    : {token,{'-=',TokenLine}}.
\*=   : {token,{'*=',TokenLine}}.
/=    : {token,{'/=',TokenLine}}.
\*\*= : {token,{'**=',TokenLine}}.
\.\.  : {token,{'..',TokenLine}}.
\|    : {token,{'|',TokenLine}}.
\|\|  : {token,{'or',TokenLine}}.
\|\|= : {token,{'||=',TokenLine}}.
&     : {token,{'&',TokenLine}}.
&&    : {token,{'and',TokenLine}}.
\^    : {token,{'^',TokenLine}}.
~     : {token,{'~',TokenLine}}.
\?    : {token,{'?',TokenLine}}.
!     : {token,{'!',TokenLine}}.
\$    : {token,{'$',TokenLine}}.
&=    : {token,{'&=',TokenLine}}.
\|=   : {token,{'|=',TokenLine}}.
\^=   : {token,{'^=',TokenLine}}.
<<=   : {token,{'<<=',TokenLine}}.
>>=   : {token,{'>>=',TokenLine}}.

Erlang code.  

-include("reia_nodes.hrl").

build_integer(Chars, Line) ->
  {token, #integer{line = Line, value =  list_to_integer(Chars)}}.

build_float(Chars, Line) ->
  {token, #float{line = Line, value =  list_to_float(Chars)}}.
  
build_string(Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 2, Len - 2)), 
  {token, {string, Line, String}}.

build_regexp(Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 4, Len - 4)),
  {token, {regexp, Line, String}}.
  
unescape_string(String) -> unescape_string(String, []).

unescape_string([], Output) ->
  lists:reverse(Output);
unescape_string([$\\, Escaped | Rest], Output) ->
  Char = case Escaped of
    $\\ -> $\\;
    $/  -> $/; 
    $\" -> $\";
    $\' -> $\';
    $b  -> $\b;
    $d  -> $\d;
    $e  -> $\e;
    $f  -> $\f;
    $n  -> $\n;
    $r  -> $\r;
{'fraction_non_alphanumeric': 0.294712376525276, 'fraction_numerical': 0.002789076118535735, 'mean_word_length': 3.9064994298745725, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7289930', 'n_tokens_mistral': 3504, 'n_tokens_neox': 3249, 'n_words': 450}