<commit_before>#include "../include/parser.h"
#include "../include/exceptions.h"

#define IGNORE_PARSE_ERROR(FUN, ARG)\
try                                 \
{                                   \
    return FUN(ARG);                \
}                                   \
catch (ParseError e)                \
{                                   \
                                    \
}                                   \


inline char get_char(FileCursor& cursor)
{
    char c = cursor.get();
    if (cursor.eof())
        throw ParseError();
    return c;
}

//   sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
//                 / "*" / "+" / "," / ";" / "="
FileCursor parse_sub_delims(FileCursor cursor)
{
    char c = get_char(cursor);
    if(c == '!' || c == '$' || c == '&' || c == '\'' || c == '(' || c == ')' ||
        c == '*' || c == '+' || c == ',' || c == ';' || c == '=')
        return cursor;
    else
        throw ParseError(); 
}

//   gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
FileCursor parse_gen_delims(FileCursor cursor)
{
    char c = get_char(cursor);
    if (c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' ||
            c =='@')
        return cursor;
    else
        throw ParseError();
}

//   reserved      = gen-delims / sub-delims
FileCursor parse_reserved(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_gen_delims, cursor);
    IGNORE_PARSE_ERROR(parse_sub_delims, cursor);
    throw ParseError();
}

FileCursor parse_alpha(FileCursor cursor)
{
    char c = get_char(cursor);
    if(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'))
        return cursor;
    else
        throw ParseError();
}

FileCursor parse_digit(FileCursor cursor)
{
    char c = get_char(cursor);
    if('0' <= c && c <= '9')
        return cursor;
    else
        throw ParseError();
}

//   unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
FileCursor parse_unreserved(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_alpha, cursor);
    IGNORE_PARSE_ERROR(parse_digit, cursor);
    char c = get_char(cursor);
    if (c == '-' || c == '.' || c == '_' || c == '~')
        return cursor;
    else
        throw ParseError();
}

FileCursor parse_hexdigit(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_digit, cursor);
    char c = get_char(cursor);
    if(('A' <= c && c <= 'F') || ('a' <= c && c <= 'f'))
        return cursor;
    throw ParseError();
}

//   pct-encoded   = "%" HEXDIG HEXDIG
FileCursor parse_pct_encoded(FileCursor cursor)
{
    char c = get_char(cursor);
    if (c != '%')
        throw ParseError();
    parse_hexdigit(cursor);
    return parse_hexdigit(cursor);
}

//   pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
FileCursor parse_pchar(FileCursor cursor)<commit_msg>Add hexdigit grammar comment.<commit_after>#include "../include/parser.h"
#include "../include/exceptions.h"

#define IGNORE_PARSE_ERROR(FUN, ARG)\
try                                 \
{                                   \
    return FUN(ARG);                \
}                                   \
catch (ParseError e)                \
{                                   \
                                    \
}                                   \


inline char get_char(FileCursor& cursor)
{
    char c = cursor.get();
    if (cursor.eof())
        throw ParseError();
    return c;
}

//   sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
//                 / "*" / "+" / "," / ";" / "="
FileCursor parse_sub_delims(FileCursor cursor)
{
    char c = get_char(cursor);
    if(c == '!' || c == '$' || c == '&' || c == '\'' || c == '(' || c == ')' ||
        c == '*' || c == '+' || c == ',' || c == ';' || c == '=')
        return cursor;
    else
        throw ParseError(); 
}

//   gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
FileCursor parse_gen_delims(FileCursor cursor)
{
    char c = get_char(cursor);
    if (c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' ||
            c =='@')
        return cursor;
    else
        throw ParseError();
}

//   reserved      = gen-delims / sub-delims
FileCursor parse_reserved(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_gen_delims, cursor);
    IGNORE_PARSE_ERROR(parse_sub_delims, cursor);
    throw ParseError();
}

FileCursor parse_alpha(FileCursor cursor)
{
    char c = get_char(cursor);
    if(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'))
        return cursor;
    else
        throw ParseError();
}

FileCursor parse_digit(FileCursor cursor)
{
    char c = get_char(cursor);
    if('0' <= c && c <= '9')
        return cursor;
    else
        throw ParseError();
}

//   unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
FileCursor parse_unreserved(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_alpha, cursor);
    IGNORE_PARSE_ERROR(parse_digit, cursor);
    char c = get_char(cursor);
    if (c == '-' || c == '.' || c == '_' || c == '~')
        return cursor;
    else
        throw ParseError();
}

// hexdigit = DIGIT / 'A' - 'F' / 'a' - 'f'
FileCursor parse_hexdigit(FileCursor cursor)
{
    IGNORE_PARSE_ERROR(parse_digit, cursor);
    char c = get_char(cursor);
    if(('A' <= c && c <= 'F') || ('a' <= c && c <= 'f'))
        return cursor;
    throw ParseError();
}

//   pct-encoded   = "%" HEXDIG HEXDIG
FileCursor parse_pct_encoded(FileCursor cursor)
{
    char c = get_char(cursor);
    if (c != '%')
        throw ParseError();
    parse_hexdigit(cursor);
    return parse_hexdigit(cursor);
}

//   pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
FileCursor parse_pchar(FileCursor cursor)
{'fraction_non_alphanumeric': 0.1932263814616756, 'fraction_numerical': 0.00071301247771836, 'mean_word_length': 1.8040979510244877, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295546', 'n_tokens_mistral': 1956, 'n_tokens_neox': 1877, 'n_words': 370}