<commit_before>%
% reia_scan: Leex scanner for the Reia language
% Copyright (C)2008 <NAME>
% 
% Redistribution is permitted under the MIT license.  See LICENSE for details.
%

Definitions.

Digit = [0-9]
UpperCase = [A-Z]
LowerCase = [a-z]
Whitespace = [\s]
DoubleQuoted = "(\\\^.|\\.|[^\"])*"
SingleQuoted = '(\\\^.|\\.|[^\'])*'
Regexp = /(\\\^.|\\.|[^/])*/
Comment = #.*

Rules.

%% Indentation
\n\s+ : build_indentation(TokenLine, TokenLen).

%% Numbers
-?({Digit}+((\'|\`|_)|{Digit}+)*)*\.({Digit}+((\'|\`|_){Digit}+)*)+ : build_float(TokenChars, TokenLine).
-?{Digit}+((\'|\`|_){Digit}+)* : build_integer(TokenChars, TokenLine).

%% Strings
{DoubleQuoted} : build_string(string, TokenChars, TokenLine, TokenLen).
{SingleQuoted} : build_string(string, TokenChars, TokenLine, TokenLen).

%% Regular expressions
{Regexp} : build_string(regexp, TokenChars, TokenLine, TokenLen).

%% Atoms
\~({UpperCase}|{LowerCase}|_)({UpperCase}|{Digit}|{LowerCase}|_)* : build_atom(TokenChars, TokenLine, TokenLen).
\~{DoubleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).
\~{SingleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).

%% Identifiers and constants
{UpperCase}({UpperCase}|{LowerCase}|{Digit}|_)* : build_constant(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)* : build_identifier(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)*[?!] : build_punctuated_identifier(TokenChars, TokenLine).

%% Ignored
{Comment} : skip_token.
{Whitespace}+ : skip_token.

\n    : {token,{eol,TokenLine}}.
\(    : {token,{'(',TokenLine}}.
\)    : {token,{')',TokenLine}}.
\[    : {token,{'[',TokenLine}}.
\]    : {token,{']',TokenLine}}.
\{    : {token,{'{',TokenLine}}.
}     : {token,{'}',TokenLine}}.
<-    : {token,{'<-',TokenLine}}.
->    : {token,{'->',TokenLine}}.
<<    : {token,{'<<',TokenLine}}.
>>    : {token,{'>>',TokenLine}}.
\+    : {token,{'+',TokenLine}}.
-     : {token,{'-',TokenLine}}.
\*    : {token,{'*',TokenLine}}.
/     : {token,{'/',TokenLine}}.
\%    : {token,{'%',TokenLine}}.
\*\*  : {token,{'**',TokenLine}}.
\.    : {token,{'.',TokenLine}}.
,     : {token,{',',TokenLine}}.
:     : {token,{':',TokenLine}}.
::    : {token,{'::',TokenLine}}.
;     : {token,{';',TokenLine}}.
@     : {token,{'@',TokenLine}}.
=     : {token,{'=',TokenLine}}.
==    : {token,{'==',TokenLine}}.
===   : {token,{'===',TokenLine}}.
!=    : {token,{'!=',TokenLine}}.
<     : {token,{'<',TokenLine}}.
>     : {token,{'>',TokenLine}}.
<=    : {token,{'<=',TokenLine}}.
>=    : {token,{'>=',TokenLine}}.
\.\.  : {token,{'..',TokenLine}}.
\|    : {token,{'|',TokenLine}}.
\|\|  : {token,{'or',TokenLine}}.
&&    : {token,{'and',TokenLine}}.
!     : {token,{'not',TokenLine}}.

Erlang code.  

-export([scan/1, scan/2, process_indentation/2]).

%% Scan the string for tokens
scan(String) ->
  scan(String, 1).
  
%% Scan the string for tokens, providing a starting line
scan(String, Line) ->
  case string(String, Line) of
    {ok, Tokens, Lines} ->
      process_indentation(Tokens, Lines);
    Result ->
      Result
  end.

%% Process indentation tokens, converting them to indent/dedent      
process_indentation(Tokens, Lines) ->
  case process_indentation(Tokens, [], [0]) of
    {ok, NewTokens, State} -> 
      {ok, eof_dedents(NewTokens, Lines, State), Lines};
    Result ->
      Result
  end.
  
%% Generate dedents for all remaining indentation levels when EOF is reached
eof_dedents(Tokens, Lines, [0]) ->
  lists:reverse(Tokens);
eof_dedents(Tokens, Lines, [_|State]) ->
  eof_dedents([{dedent, Lines}|Tokens], Lines, State).

%% Indentation processing finished
process_indentation([], Tokens, State) -> 
  {ok, Tokens, State};

%% Ignore blank lines
process_indentation([{indentation, _, _}, Token = {indentation, _, _}|Rest], Tokens, State) ->
  process_indentation([Token|Rest], Tokens, State);
process_indentation([{indentation, Line, _}, Token = {eol, _}|Rest], Tokens, State) ->
  process_indentation([{eol, Line}|Rest], Tokens, State);
process_indentation([Token = {eol, _}, {eol, _}|Rest], Tokens, State) ->
  process_indentation([Token|Rest], Tokens, State);
      
%% Convert EOL tokens to indentation tokens
process_indentation([{eol, Line}|Rest], Tokens, State) ->
  process_indentation([{indentation, Line, 0}|Rest], Tokens, State);
  
%% Handle indents or dedents  
process_indentation([{indentation, Line, Amount}|Rest], Tokens, State = [Current|_]) ->
  if
    Amount == Current ->
      process_indentation(Rest, [{eol, Line - 1}|Tokens], State);
    Amount > Current ->
      process_indentation(Rest, [{indent, Line},{eol, Line - 1}|Tokens], [Amount|State]);
    Amount < Current ->
      case build_dedent([{eol, Line - 1}|Tokens], Amount, Line, State) of
        {ok, NewTokens, NewState} ->
          process_indentation(Rest, NewTokens, NewState);
        nomatch ->
          {error, {Line, ?MODULE,{illegal,dedent}},Line}
      end
  end;
  
%% Pass through all other tokens
process_indentation([Token|Rest], Tokens, State) ->
  process_indentation(Rest, [Token|Tokens], State).
  
build_dedent(Tokens, Amount, Line, [Current|State]) ->
  if
    Amount == Current ->
      {ok, Tokens, [Current|State]};
    Amount > Current ->
      nomatch;  
    true ->
      build_dedent([{dedent, Line}|Tokens], Amount, Line, State)
  end.
  
build_indentation(Line, Length) ->
  {token, {indentation, Line, Length - 1}}.

remove_cosmetics(Chars, CharsToRemove) ->
  [S || S <- Chars, [A || A <- CharsToRemove, S =:= A] =:= []].

build_integer([$-|Chars], Line) ->
  {token, {integer, Line, -list_to_integer(remove_cosmetics(Chars, [$', $_, $`]))}};
build_integer(Chars, Line) ->
  {token, {integer, Line, list_to_integer(remove_cosmetics(Chars, [$', $_, $`]))}}.

build_float([$-|Chars], Line) ->
  {token, {float, Line, -list_to_float(remove_cosmetics(Chars, [$', $_, $`]))}};
build_float(Chars, Line) ->
  {token, {float, Line, list_to_float(remove_cosmetics(Chars, [$', $_, $`]))}}.

build_string(Type, Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 2, Len - 2)), 
  {token, {Type, Line, String}}.
  
unescape_string(String) -> unescape_string(String, []).

unescape_string([], Output) ->
  lists:reverse(Output);
unescape_string([$\\, Escaped | Rest], Output) ->
  Char = case Escaped of
    $\\ -> $\\;
    $\" -> $\";
    $\' -> $\';
    $b  -> $\b;
    $d  -> $\d;
    $e  -> $\e;<commit_msg>Reverted delimiters in numbers for now<commit_after>%
% reia_scan: Leex scanner for the Reia language
% Copyright (C)2008 <NAME>
% 
% Redistribution is permitted under the MIT license.  See LICENSE for details.
%

Definitions.

Digit = [0-9]
UpperCase = [A-Z]
LowerCase = [a-z]
Whitespace = [\s]
DoubleQuoted = "(\\\^.|\\.|[^\"])*"
SingleQuoted = '(\\\^.|\\.|[^\'])*'
Regexp = /(\\\^.|\\.|[^/])*/
Comment = #.*

Rules.

%% Indentation
\n\s+ : build_indentation(TokenLine, TokenLen).

%% Numbers
-?{Digit}+\.{Digit}+ : build_float(TokenChars, TokenLine).
-?{Digit}+ : build_integer(TokenChars, TokenLine).

%% Strings
{DoubleQuoted} : build_string(string, TokenChars, TokenLine, TokenLen).
{SingleQuoted} : build_string(string, TokenChars, TokenLine, TokenLen).

%% Regular expressions
{Regexp} : build_string(regexp, TokenChars, TokenLine, TokenLen).

%% Atoms
\~({UpperCase}|{LowerCase}|_)({UpperCase}|{Digit}|{LowerCase}|_)* : build_atom(TokenChars, TokenLine, TokenLen).
\~{DoubleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).
\~{SingleQuoted} : build_quoted_atom(TokenChars, TokenLine, TokenLen).

%% Identifiers and constants
{UpperCase}({UpperCase}|{LowerCase}|{Digit}|_)* : build_constant(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)* : build_identifier(TokenChars, TokenLine).
({LowerCase}|_)({UpperCase}|{LowerCase}|{Digit}|_)*[?!] : build_punctuated_identifier(TokenChars, TokenLine).

%% Ignored
{Comment} : skip_token.
{Whitespace}+ : skip_token.

\n    : {token,{eol,TokenLine}}.
\(    : {token,{'(',TokenLine}}.
\)    : {token,{')',TokenLine}}.
\[    : {token,{'[',TokenLine}}.
\]    : {token,{']',TokenLine}}.
\{    : {token,{'{',TokenLine}}.
}     : {token,{'}',TokenLine}}.
<-    : {token,{'<-',TokenLine}}.
->    : {token,{'->',TokenLine}}.
<<    : {token,{'<<',TokenLine}}.
>>    : {token,{'>>',TokenLine}}.
\+    : {token,{'+',TokenLine}}.
-     : {token,{'-',TokenLine}}.
\*    : {token,{'*',TokenLine}}.
/     : {token,{'/',TokenLine}}.
\%    : {token,{'%',TokenLine}}.
\*\*  : {token,{'**',TokenLine}}.
\.    : {token,{'.',TokenLine}}.
,     : {token,{',',TokenLine}}.
:     : {token,{':',TokenLine}}.
::    : {token,{'::',TokenLine}}.
;     : {token,{';',TokenLine}}.
@     : {token,{'@',TokenLine}}.
=     : {token,{'=',TokenLine}}.
==    : {token,{'==',TokenLine}}.
===   : {token,{'===',TokenLine}}.
!=    : {token,{'!=',TokenLine}}.
<     : {token,{'<',TokenLine}}.
>     : {token,{'>',TokenLine}}.
<=    : {token,{'<=',TokenLine}}.
>=    : {token,{'>=',TokenLine}}.
\.\.  : {token,{'..',TokenLine}}.
\|    : {token,{'|',TokenLine}}.
\|\|  : {token,{'or',TokenLine}}.
&&    : {token,{'and',TokenLine}}.
!     : {token,{'not',TokenLine}}.

Erlang code.  

-export([scan/1, scan/2, process_indentation/2]).

%% Scan the string for tokens
scan(String) ->
  scan(String, 1).
  
%% Scan the string for tokens, providing a starting line
scan(String, Line) ->
  case string(String, Line) of
    {ok, Tokens, Lines} ->
      process_indentation(Tokens, Lines);
    Result ->
      Result
  end.

%% Process indentation tokens, converting them to indent/dedent      
process_indentation(Tokens, Lines) ->
  case process_indentation(Tokens, [], [0]) of
    {ok, NewTokens, State} -> 
      {ok, eof_dedents(NewTokens, Lines, State), Lines};
    Result ->
      Result
  end.
  
%% Generate dedents for all remaining indentation levels when EOF is reached
eof_dedents(Tokens, Lines, [0]) ->
  lists:reverse(Tokens);
eof_dedents(Tokens, Lines, [_|State]) ->
  eof_dedents([{dedent, Lines}|Tokens], Lines, State).

%% Indentation processing finished
process_indentation([], Tokens, State) -> 
  {ok, Tokens, State};

%% Ignore blank lines
process_indentation([{indentation, _, _}, Token = {indentation, _, _}|Rest], Tokens, State) ->
  process_indentation([Token|Rest], Tokens, State);
process_indentation([{indentation, Line, _}, Token = {eol, _}|Rest], Tokens, State) ->
  process_indentation([{eol, Line}|Rest], Tokens, State);
process_indentation([Token = {eol, _}, {eol, _}|Rest], Tokens, State) ->
  process_indentation([Token|Rest], Tokens, State);
      
%% Convert EOL tokens to indentation tokens
process_indentation([{eol, Line}|Rest], Tokens, State) ->
  process_indentation([{indentation, Line, 0}|Rest], Tokens, State);
  
%% Handle indents or dedents  
process_indentation([{indentation, Line, Amount}|Rest], Tokens, State = [Current|_]) ->
  if
    Amount == Current ->
      process_indentation(Rest, [{eol, Line - 1}|Tokens], State);
    Amount > Current ->
      process_indentation(Rest, [{indent, Line},{eol, Line - 1}|Tokens], [Amount|State]);
    Amount < Current ->
      case build_dedent([{eol, Line - 1}|Tokens], Amount, Line, State) of
        {ok, NewTokens, NewState} ->
          process_indentation(Rest, NewTokens, NewState);
        nomatch ->
          {error, {Line, ?MODULE,{illegal,dedent}},Line}
      end
  end;
  
%% Pass through all other tokens
process_indentation([Token|Rest], Tokens, State) ->
  process_indentation(Rest, [Token|Tokens], State).
  
build_dedent(Tokens, Amount, Line, [Current|State]) ->
  if
    Amount == Current ->
      {ok, Tokens, [Current|State]};
    Amount > Current ->
      nomatch;  
    true ->
      build_dedent([{dedent, Line}|Tokens], Amount, Line, State)
  end.
  
build_indentation(Line, Length) ->
  {token, {indentation, Line, Length - 1}}.

build_integer([$-|Chars], Line) ->
  {token, {integer, Line, -list_to_integer(Chars)}};
build_integer(Chars, Line) ->
  {token, {integer, Line, list_to_integer(Chars)}}.

build_float([$-|Chars], Line) ->
  {token, {float, Line, -list_to_float(Chars)}};
build_float(Chars, Line) ->
  {token, {float, Line, list_to_float(Chars)}}.
  
build_string(Type, Chars, Line, Len) ->
  String = unescape_string(lists:sublist(Chars, 2, Len - 2)), 
  {token, {Type, Line, String}}.
  
unescape_string(String) -> unescape_string(String, []).

unescape_string([], Output) ->
  lists:reverse(Output);
unescape_string([$\\, Escaped | Rest], Output) ->
  Char = case Escaped of
    $\\ -> $\\;
    $\" -> $\";
    $\' -> $\';
    $b  -> $\b;
    $d  -> $\d;
    $e  -> $\e;
{'fraction_non_alphanumeric': 0.22568310364056401, 'fraction_numerical': 0.0030271648211582887, 'mean_word_length': 4.10948310948311, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 110, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3920159', 'n_tokens_mistral': 4650, 'n_tokens_neox': 4431, 'n_words': 838}