<commit_before>//
//  yl_parser_ragel.rl
//
//  Created by <NAME> on 14/03/2014.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//


/*
    ragel yl_parser_ragel.rl -o yl_parser_ragel.cpp -G2
*/


#include "yl_parser.h"
#include <stdint.h>
#include <intformat.h>
#include <unordered_map>
#include <region.h>
#include "yl_token.h"
#include "yl_diagnostics.h"


#pragma GCC diagnostic ignored "-Wunreachable-code"


void* XecParseAlloc( void* (*malloc)( size_t ) );
void  XecParse( void* yyp, int yymajor, yl_token* yyminor, yl_parser* p );
void  XecParseFree( void* p, void (*free)( void* ) );



%%{

    machine lexer;
    
    alphtype unsigned char;
    
    
    action nl { diagnostics->newline( (int)( offset + ( p - buffer ) - 1 ) ); }
    action ts { sloc = (int)( offset + ( p - buffer ) ); data.clear(); }
    action dc { data.append( (char)fc ); }
    
    action neof
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        diagnostics->error( sloc, "unexpected end of file" );
        fbreak;
    }
    
    action nchar
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        if ( fc >= 0x20 && fc <= 0x7E )
            diagnostics->error( sloc, "unexpected character '%c'", fc );
        else
            diagnostics->error( sloc, "unexpected character '\\x%02X'", fc );
        if ( diagnostics->error_count() >= ERROR_LIMIT )
            goto error;
    }


    
    # Newlines (handling CR LF properly), comments, and whitespace.

    newline         = ( '\n' | '\r' | '\r\n' ) $1 %0 %nl %!nl ;
    block_comment   = '/*' ( any | newline )* :>> '*/' ;
    line_comment    = '//' any* :>> newline ;
    whitespace      = [ \t] | newline | block_comment | line_comment ;


    identifier =
    (
            [a-zA-Z_][a-zA-Z0-9_]*
    ) $dc;

    
    number =
    (
            ( '0' | [1-9][0-9]* ) ( '.' [0-9]+ )? ( [eE] ( '+' | '-' )? [0-9]+ )?
        |   '0x' [0-9A-Fa-f]+ ( '.' [0-9A-Fa-f]+ )? ( [pP] ( '+' | '-' )? [0-9]+ )?<commit_msg>Allow 0X-prefixed hex numbers (since I support E and P...)<commit_after>//
//  yl_parser_ragel.rl
//
//  Created by <NAME> on 14/03/2014.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//


/*
    ragel yl_parser_ragel.rl -o yl_parser_ragel.cpp -G2
*/


#include "yl_parser.h"
#include <stdint.h>
#include <intformat.h>
#include <unordered_map>
#include <region.h>
#include "yl_token.h"
#include "yl_diagnostics.h"


#pragma GCC diagnostic ignored "-Wunreachable-code"


void* XecParseAlloc( void* (*malloc)( size_t ) );
void  XecParse( void* yyp, int yymajor, yl_token* yyminor, yl_parser* p );
void  XecParseFree( void* p, void (*free)( void* ) );



%%{

    machine lexer;
    
    alphtype unsigned char;
    
    
    action nl { diagnostics->newline( (int)( offset + ( p - buffer ) - 1 ) ); }
    action ts { sloc = (int)( offset + ( p - buffer ) ); data.clear(); }
    action dc { data.append( (char)fc ); }
    
    action neof
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        diagnostics->error( sloc, "unexpected end of file" );
        fbreak;
    }
    
    action nchar
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        if ( fc >= 0x20 && fc <= 0x7E )
            diagnostics->error( sloc, "unexpected character '%c'", fc );
        else
            diagnostics->error( sloc, "unexpected character '\\x%02X'", fc );
        if ( diagnostics->error_count() >= ERROR_LIMIT )
            goto error;
    }


    
    # Newlines (handling CR LF properly), comments, and whitespace.

    newline         = ( '\n' | '\r' | '\r\n' ) $1 %0 %nl %!nl ;
    block_comment   = '/*' ( any | newline )* :>> '*/' ;
    line_comment    = '//' any* :>> newline ;
    whitespace      = [ \t] | newline | block_comment | line_comment ;


    identifier =
    (
            [a-zA-Z_][a-zA-Z0-9_]*
    ) $dc;

    
    number =
    (
            ( '0' | [1-9][0-9]* ) ( '.' [0-9]+ )? ( [eE] ( '+' | '-' )? [0-9]+ )?
        |   '0' [xX] [0-9A-Fa-f]+ ( '.' [0-9A-Fa-f]+ )? ( [pP] ( '+' | '-' )? [0-9]+ )?
{'fraction_non_alphanumeric': 0.18955042527339003, 'fraction_numerical': 0.02017010935601458, 'mean_word_length': 2.0331613854089903, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '235979', 'n_tokens_mistral': 1634, 'n_tokens_neox': 1562, 'n_words': 358}