<commit_before>//..............................................................................
//
//  This file is part of the AXL library.
//
//  AXL is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/axl/license.txt
//
//..............................................................................

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace axl {
namespace st {

//..............................................................................

%%{

machine axl_lua_st;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# standard definitions
#

dec    = [0-9];
hex    = [0-9a-fA-F];
oct    = [0-7];
bin    = [01];
id     = [a-zA-Z_] [a-zA-Z0-9_]*;
ws     = [ \t\r]+;
nl     = '\n' @{ newLine (p + 1); };
esc    = '\\' [^\n];
lit_dq = '"' ([^"\n\\] | esc)* (["\\] | nl);
lit_sq = "'" ([^'\n\\] | esc)* (['\\] | nl);

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

user_data := |*

[(){}]       { createToken (ts [0]); };
nl           ;
any          ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

user_code := |*

'{'           { createToken ('{'); };
'}' (ws* nl)? { createToken ('}'); };
nl            ;
any           ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

main := |*

'%{'         { createToken (TokenKind_OpenCode); };
'${'         { createToken (TokenKind_OpenData_c); };
'$('         { createToken (TokenKind_OpenData_r); };
'$' id       { createStringToken (TokenKind_Data, 1); };
nl           ;
any          ;

*|;

}%%

//..............................................................................

void
Lexer::init ()
{
	%% write init;
}

void
Lexer::exec ()
{
	%% write exec;
}

int
Lexer::getMachineState (LexerMachineKind machine)
{
	switch (machine)<commit_msg>[axl_st] added literals to user_code and user_data machines<commit_after>//..............................................................................
//
//  This file is part of the AXL library.
//
//  AXL is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/axl/license.txt
//
//..............................................................................

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace axl {
namespace st {

//..............................................................................

%%{

machine axl_lua_st;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# standard definitions
#

dec    = [0-9];
hex    = [0-9a-fA-F];
oct    = [0-7];
bin    = [01];
id     = [a-zA-Z_] [a-zA-Z0-9_]*;
ws     = [ \t\r]+;
nl     = '\n' @{ newLine (p + 1); };
esc    = '\\' [^\n];
lit_dq = '"' ([^"\n\\] | esc)* (["\\] | nl);
lit_sq = "'" ([^'\n\\] | esc)* (['\\] | nl);

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

user_data := |*

[(){}]        { createToken (ts [0]); };
lit_dq        ;
lit_sq        ;
nl            ;
any           ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

user_code := |*

'{'           { createToken ('{'); };
'}' (ws* nl)? { createToken ('}'); };
lit_dq        ;
lit_sq        ;
nl            ;
any           ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

main := |*

'%{'          { createToken (TokenKind_OpenCode); };
'${'          { createToken (TokenKind_OpenData_c); };
'$('          { createToken (TokenKind_OpenData_r); };
'$' id        { createStringToken (TokenKind_Data, 1); };
nl            ;
any           ;

*|;

}%%

//..............................................................................

void
Lexer::init ()
{
	%% write init;
}

void
Lexer::exec ()
{
	%% write exec;
}

int
Lexer::getMachineState (LexerMachineKind machine)
{
	switch (machine)
{'fraction_non_alphanumeric': 0.3714353052085788, 'fraction_numerical': 0.009898656610888523, 'mean_word_length': 0.9317250796540737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5550950', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1489, 'n_words': 258}