<commit_before>#ifndef DICE_LEXER_HPP_
#define DICE_LEXER_HPP_

#include <string>
#include <sstream>
#include <cctype>
#include <istream>

#include "logger.hpp"

namespace dice 
{
    enum class token_type
    {
        number_fp,
        number_int,
        id,
        rel_op,
        roll_op,
        add,
        sub,
        mult,
        div,
        left_parent,
        right_parent,
        left_square_bracket,
        right_square_bracket,
        param_delim,
        in,
        end
    };

    struct token
    {
        token_type type;
        std::string value;

        token() {}
        token(token_type type) : type(type) {}
        token(token_type type, std::string value) :
            type(type),
            value(std::move(value)) {}
    };

    // convert token type to a human readable string
    std::string to_string(token_type type);

    // convert token value and type to a human readable string
    std::string to_string(const token& token);

    // location in the input stream
    struct lexer_location
    {
        int line;
        int col;

        lexer_location() : line(0), col(0) {}
        lexer_location(int line, int col = 0) : line(line), col(col) {}
    };

    class lexer 
    {
    public:
        lexer(std::istream* input, logger* log);

        // disallow copy so that 2 lexers don't read from the same input
        lexer(const lexer&) = delete;
        void operator=(const lexer&) = delete;

        // allow move
        lexer(lexer&&) = default;
        lexer& operator=(lexer&&) = default;

        /** Read token at current location.
         * Decode a sequance of characters as a token and 
         * advance in the input stream.
         * @return token at current position
         */
        token read_token();

        // Return current location of the lexer in the input stream
        inline const lexer_location& location() const <commit_msg>Use initializer insted of constructor<commit_after>#ifndef DICE_LEXER_HPP_
#define DICE_LEXER_HPP_

#include <string>
#include <sstream>
#include <cctype>
#include <istream>

#include "logger.hpp"

namespace dice 
{
    enum class token_type
    {
        number_fp,
        number_int,
        id,
        rel_op,
        roll_op,
        add,
        sub,
        mult,
        div,
        left_parent,
        right_parent,
        left_square_bracket,
        right_square_bracket,
        param_delim,
        in,
        end
    };

    struct token
    {
        token_type type;
        std::string value;

        token() {}
        token(token_type type) : type(type) {}
        token(token_type type, std::string value) :
            type(type),
            value(std::move(value)) {}
    };

    // convert token type to a human readable string
    std::string to_string(token_type type);

    // convert token value and type to a human readable string
    std::string to_string(const token& token);

    // location in the input stream
    struct lexer_location
    {
        int line = 0;
        int col = 0;
    };

    class lexer 
    {
    public:
        lexer(std::istream* input, logger* log);

        // disallow copy so that 2 lexers don't read from the same input
        lexer(const lexer&) = delete;
        void operator=(const lexer&) = delete;

        // allow move
        lexer(lexer&&) = default;
        lexer& operator=(lexer&&) = default;

        /** Read token at current location.
         * Decode a sequance of characters as a token and 
         * advance in the input stream.
         * @return token at current position
         */
        token read_token();

        // Return current location of the lexer in the input stream
        inline const lexer_location& location() const 
{'fraction_non_alphanumeric': 0.08967173738991192, 'fraction_numerical': 0.0018681611956231651, 'mean_word_length': 1.9167315175097277, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6108670', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1073, 'n_words': 338}