<commit_before>//
//  orbit/parser/compiler_utils.c
//  Orbit - Parser
//
//  Created by <NAME> on 2017-05-21.
//  Copyright © 2017 <NAME>. All rights reserved.
//
#include <orbit/parser/compiler_utils.h>

typedef struct {
    const char* name;
    const char* string;
    bool        isBinaryOp;
    bool        isUnaryOp;
} OCTokenData;

static const OCTokenData _tokenData[] = {
    [TOKEN_LPAREN] = {"l_paren", "(", true, false},
    [TOKEN_RPAREN] = {"r_paren", ")", false, false},
    [TOKEN_LBRACE] = {"l_brace", "{", false, false},
    [TOKEN_RBRACE] = {"r_brace", "}", false, false},
    [TOKEN_LBRACKET] = {"l_bracket", "[", true, false},
    [TOKEN_RBRACKET] = {"r_bracket", "]", false, false},
    [TOKEN_PLUS] = {"plus", "+", true, false},
    [TOKEN_MINUS] = {"plus", "-", true, true},
    [TOKEN_SLASH] = {"slash", "/", true, false},
    [TOKEN_STAR] = {"star", "*", true, false},
    [TOKEN_STARSTAR] = {"starstar", "**", true, false},
    [TOKEN_PERCENT] = {"percent", "%", true, false},
    [TOKEN_CARET] = {"carent", "^", true, false},
    [TOKEN_TILDE] = {"tilde", "~", false, true},
    [TOKEN_AMP] = {"amp", "&", true, false},
    [TOKEN_PIPE] = {"pipe", "|", true, false},
    [TOKEN_BANG] = {"exclaim", "!", false, true},
    [TOKEN_QUESTION] = {"question", "?", false, false},
    [TOKEN_LT] = {"less", "<", true, false},
    [TOKEN_GT] = {"greater", ">", true, false},
    [TOKEN_EQUALS] = {"equal", "=", true, false},
    [TOKEN_LTEQ] = {"less_equal", "<=", true, false},
    [TOKEN_GTEQ] = {"greater_equal", ">=", true, false},
    [TOKEN_EQEQ] = {"equal_equal", "==", true, false},
    [TOKEN_PLUSEQ] = {"plus_equal", "+=", true, false},
    [TOKEN_MINUSEQ] = {"minus_equal", "-=", true, false},
    [TOKEN_STAREQ] = {"star_equal", "*=", true, false},
    [TOKEN_SLASHEQ] = {"slash_equal", "/=", true, false},
    [TOKEN_BANGEQ] = {"exclaim_equal", "!=", true, false},
    [TOKEN_LTLT] = {"less_less", "<<", true, false},
    [TOKEN_GTGT] = {"greater_greater", ">>", true, false},
    [TOKEN_AMPAMP] = {"amp_amp", "&&", true, false},
    [TOKEN_PIPEPIPE] = {"pipe_pipe", "||", true, false},
    [TOKEN_SEMICOLON] = {"semicolon", ";", false, false},
    [TOKEN_NEWLINE] = {"newline", "\\n", false, false},
    [TOKEN_COLON] = {"colon", ":", false, false},
    [TOKEN_COMMA] = {"comma", ",", false, false},
    [TOKEN_DOT] = {"dot", ".", true, false},
    [TOKEN_ARROW] = {"arrow", "->", false, false},
    [TOKEN_INTEGER_LITERAL] = {"integer_constant", "integer constant", false, false},
    [TOKEN_FLOAT_LITERAL] = {"float_constant", "floating-point constant", false, false},
    [TOKEN_STRING_LITERAL] = {"string_constant", "string constant", false, false},
    [TOKEN_IDENTIFIER] = {"raw_identifier", "identifier", false, false},
    [TOKEN_FUN] = {"fun_kw", "fun", false, false},
    [TOKEN_VAR] = {"var_kw", "var", false, false},
    [TOKEN_CONST] = {"const_kw", "const", false, false},
    [TOKEN_MAYBE] = {"maybe_kw", "maybe", false, false},
    [TOKEN_TYPE] = {"type_kw", "type", false, false},
    [TOKEN_RETURN] = {"return_kw", "return", false, false},
    [TOKEN_FOR] = {"for_kw", "for", false, false},
    [TOKEN_WHILE] = {"while_kw", "while", false, false},
    [TOKEN_BREAK] = {"break_kw", "break", false, false},
    [TOKEN_IN] = {"in_kw", "in", false, false},
    [TOKEN_IF] = {"if_kw", "if", false, false},
    [TOKEN_ELSE] = {"else_kw", "else", false, false},
    [TOKEN_INIT] = {"init_kw", "init", false, false},
    [TOKEN_FAIL] = {"fail_kw", "fail", false, false},
    [TOKEN_RANGE] = {"range_kw", "range", false, false},
    [TOKEN_NUMBER] = {"number_kw", "Number", false, false},
    [TOKEN_BOOL] = {"bool_kw", "Bool", false, false},
    [TOKEN_STRING] = {"string_kw", "String", false, false},
    [TOKEN_NIL] = {"nil_Kw", "Nil", false, false},
    [TOKEN_VOID] = {"void_kw", "Void", false, false},
    [TOKEN_ANY] = {"any_kw", "Any", false, false},
    [TOKEN_EOF] = {"end_of_file", "end of file", false, false},
    [TOKEN_INVALID] = {"invalid", "invalid token", false, false},
};

const char* orbit_tokenName(OCTokenType token) {
    if(token > TOKEN_INVALID) { token = TOKEN_INVALID; }<commit_msg>Fix missing tokens from dump<commit_after>//
//  orbit/parser/compiler_utils.c
//  Orbit - Parser
//
//  Created by <NAME> on 2017-05-21.
//  Copyright © 2017 <NAME>. All rights reserved.
//
#include <orbit/parser/compiler_utils.h>

typedef struct {
    const char* name;
    const char* string;
    bool        isBinaryOp;
    bool        isUnaryOp;
} OCTokenData;

static const OCTokenData _tokenData[] = {
    [TOKEN_LPAREN] = {"l_paren", "(", true, false},
    [TOKEN_RPAREN] = {"r_paren", ")", false, false},
    [TOKEN_LBRACE] = {"l_brace", "{", false, false},
    [TOKEN_RBRACE] = {"r_brace", "}", false, false},
    [TOKEN_LBRACKET] = {"l_bracket", "[", true, false},
    [TOKEN_RBRACKET] = {"r_bracket", "]", false, false},
    [TOKEN_PLUS] = {"plus", "+", true, false},
    [TOKEN_MINUS] = {"plus", "-", true, true},
    [TOKEN_SLASH] = {"slash", "/", true, false},
    [TOKEN_STAR] = {"star", "*", true, false},
    [TOKEN_STARSTAR] = {"starstar", "**", true, false},
    [TOKEN_PERCENT] = {"percent", "%", true, false},
    [TOKEN_CARET] = {"caret", "^", true, false},
    [TOKEN_TILDE] = {"tilde", "~", false, true},
    [TOKEN_AMP] = {"amp", "&", true, false},
    [TOKEN_PIPE] = {"pipe", "|", true, false},
    [TOKEN_BANG] = {"exclaim", "!", false, true},
    [TOKEN_QUESTION] = {"question", "?", false, false},
    [TOKEN_LT] = {"less", "<", true, false},
    [TOKEN_GT] = {"greater", ">", true, false},
    [TOKEN_EQUALS] = {"equal", "=", true, false},
    [TOKEN_LTEQ] = {"less_equal", "<=", true, false},
    [TOKEN_GTEQ] = {"greater_equal", ">=", true, false},
    [TOKEN_EQEQ] = {"equal_equal", "==", true, false},
    [TOKEN_PLUSEQ] = {"plus_equal", "+=", true, false},
    [TOKEN_MINUSEQ] = {"minus_equal", "-=", true, false},
    [TOKEN_STAREQ] = {"star_equal", "*=", true, false},
    [TOKEN_SLASHEQ] = {"slash_equal", "/=", true, false},
    [TOKEN_BANGEQ] = {"exclaim_equal", "!=", true, false},
    [TOKEN_LTLT] = {"less_less", "<<", true, false},
    [TOKEN_GTGT] = {"greater_greater", ">>", true, false},
    [TOKEN_AMPAMP] = {"amp_amp", "&&", true, false},
    [TOKEN_PIPEPIPE] = {"pipe_pipe", "||", true, false},
    [TOKEN_SEMICOLON] = {"semicolon", ";", false, false},
    [TOKEN_NEWLINE] = {"newline", "\\n", false, false},
    [TOKEN_COLON] = {"colon", ":", false, false},
    [TOKEN_COMMA] = {"comma", ",", false, false},
    [TOKEN_DOT] = {"dot", ".", true, false},
    [TOKEN_ARROW] = {"arrow", "->", false, false},
    [TOKEN_INTEGER_LITERAL] = {"integer_constant", "integer constant", false, false},
    [TOKEN_FLOAT_LITERAL] = {"float_constant", "floating-point constant", false, false},
    [TOKEN_STRING_LITERAL] = {"string_constant", "string constant", false, false},
    [TOKEN_IDENTIFIER] = {"raw_identifier", "identifier", false, false},
    [TOKEN_FUN] = {"fun_kw", "fun", false, false},
    [TOKEN_VAR] = {"var_kw", "var", false, false},
    [TOKEN_CONST] = {"const_kw", "const", false, false},
    [TOKEN_MAYBE] = {"maybe_kw", "maybe", false, false},
    [TOKEN_TYPE] = {"type_kw", "type", false, false},
    [TOKEN_RETURN] = {"return_kw", "return", false, false},
    [TOKEN_FOR] = {"for_kw", "for", false, false},
    [TOKEN_WHILE] = {"while_kw", "while", false, false},
    [TOKEN_BREAK] = {"break_kw", "break", false, false},
    [TOKEN_IN] = {"in_kw", "in", false, false},
    [TOKEN_IF] = {"if_kw", "if", false, false},
    [TOKEN_ELSE] = {"else_kw", "else", false, false},
    [TOKEN_INIT] = {"init_kw", "init", false, false},
    [TOKEN_FAIL] = {"fail_kw", "fail", false, false},
    [TOKEN_RANGE] = {"range_kw", "range", false, false},
    [TOKEN_NUMBER] = {"number_kw", "Number", false, false},
    [TOKEN_BOOL] = {"bool_kw", "Bool", false, false},
    [TOKEN_STRING] = {"string_kw", "String", false, false},
    [TOKEN_NIL] = {"nil_kw", "Nil", false, false},
    [TOKEN_VOID] = {"void_kw", "Void", false, false},
    [TOKEN_ARRAY] = {"array_kw", "Array", false, false},
    [TOKEN_MAP] = {"map_kw", "Map", false, false},
    [TOKEN_ANY] = {"any_kw", "Any", false, false},
    [TOKEN_EOF] = {"end_of_file", "end of file", false, false},
    [TOKEN_INVALID] = {"invalid", "invalid token", false, false},
};

const char* orbit_tokenName(OCTokenType token) {
    if(token > TOKEN_INVALID) { token = TOKEN_INVALID; }
{'fraction_non_alphanumeric': 0.2347495846190363, 'fraction_numerical': 0.0028483266081177306, 'mean_word_length': 4.179471419791026, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7631846', 'n_tokens_mistral': 3162, 'n_tokens_neox': 3237, 'n_words': 696}