<commit_before>#include <string.h> // memset, strlen
#include <stdlib.h> // malloc, free
#include <limits.h> // INT_MAX, UINT_MAX

#include "lexer.h"

namespace glsl {

// Lookup table of keywords
#undef KEYWORD
#define KEYWORD(X) { #X, kKeyword_##X },
static const keywordInfo kKeywords[] = {
    #include "lexemes.h"
};
#undef KEYWORD
#define KEYWORD(...)

// Lookup table of operators
#undef OPERATOR
#define OPERATOR(X, S, PREC) { #X, S, PREC },
static const operatorInfo kOperators[] = {
    #include "lexemes.h"
};
#undef OPERATOR
#define OPERATOR(...)

token::token() {
    memset(this, 0, sizeof *this);
}

int token::precedence() const {
    if (m_type == kType_operator)
        return kOperators[asOperator].precedence;
    return -1;
}

/// location
location::location()
    : column(1)
    , line(1)
    , position(0)
{
}

void location::advanceColumn(size_t count) {
    column += count;
    position += count;
}

void location::advanceLine() {
    line++;
    position++;
    column = 1;
}

static inline bool isDigit(int ch) {
    return unsigned(ch) - '0' < 10;
}

static inline bool isChar(int ch) {
    return (unsigned(ch) | 32) - 'a' < 26;
}

static inline bool isOctal(int ch) {
    return unsigned(ch) - '0' < 8;
}

static inline bool isHex(int ch) {
    return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || isDigit(ch);
}

static inline bool isSpace(int ch) {
    return (ch >= '\t' && ch <= '\r') || ch == ' ';
}

lexer::lexer(const char *string)
    : m_data(string)
    , m_length(0)
    , m_error(0)
{
    if (m_data)
        m_length = strlen(m_data);
}

int lexer::at(int offset) const {
    if (position() + offset < m_length)
        return m_data[position() + offset];
    return 0;
}

void lexer::read(token &out) {
    // Any previous identifier must be freed
    if (out.m_type == kType_identifier)
        free(out.asIdentifier);

    // TODO: Line continuation (backslash `\'.)
    if (position() == m_length) {
        out.m_type = kType_eof;
        return;
    }

    int ch1 = at(1);
    int ch2 = at(2);

    // Lex numerics
    if (isDigit(at()) || (at() == '.' && isDigit(ch1)))
    {
        bool isFloat = false;
        bool isDouble = false;
        bool isUnsigned = false;
        bool isOctalish = false;
        bool isHexish = false;

        if (at() == '0') {
            if (ch1 && (ch1 == 'x' || ch1 == 'X')) {
                isHexish = true;
                m_location.advanceColumn(2);
            } else {
                isOctalish = true;
            }
        }

        vector<char> numeric = readNumeric(isOctalish, isHexish);
        if (position() != m_length && at() == '.') {
            isFloat = true;
            numeric.push_back('.');
            m_location.advanceColumn();
            vector<char> others = readNumeric(isOctalish, isHexish);
            numeric.reserve(numeric.size() + others.size());
            numeric.insert(numeric.end(), others.begin(), others.end());
        }

        if (position() != m_length && (at() == 'e' || at() == 'E')) {
            ch1 = at(1);
            ch2 = at(2);
            if ((ch1 == '+' || ch1 == '-') && (ch2 >= '0' && ch2 <= '9')) {
                numeric.push_back(ch1);
                numeric.push_back(ch2);
                m_location.advanceColumn(2);
                vector<char> others = readNumeric(isOctalish, isHexish);
                numeric.reserve(numeric.size() + others.size());
                numeric.insert(numeric.end(), others.begin(), others.end());
                isFloat = true;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
        }

        if (position() != m_length && isChar(at())) {
            ch1 = at(1);
            if (at() == 'f' || at() == 'F') {
                isFloat = true;
            } else if ((at() == 'l' && ch1 == 'f') || (at() == 'L' && ch1 == 'F')) {
                isFloat = false;
                isDouble = true;
                m_location.advanceColumn();
            } else if (at() == 'u' || at() == 'U') {
                if (isFloat) {
                    m_error = "invalid use of suffix on literal";<commit_msg>fix issue #10<commit_after>#include <string.h> // memset, strlen
#include <stdlib.h> // malloc, free
#include <limits.h> // INT_MAX, UINT_MAX

#include "lexer.h"

namespace glsl {

// Lookup table of keywords
#undef KEYWORD
#define KEYWORD(X) { #X, kKeyword_##X },
static const keywordInfo kKeywords[] = {
    #include "lexemes.h"
};
#undef KEYWORD
#define KEYWORD(...)

// Lookup table of operators
#undef OPERATOR
#define OPERATOR(X, S, PREC) { #X, S, PREC },
static const operatorInfo kOperators[] = {
    #include "lexemes.h"
};
#undef OPERATOR
#define OPERATOR(...)

token::token() {
    memset(this, 0, sizeof *this);
}

int token::precedence() const {
    if (m_type == kType_operator)
        return kOperators[asOperator].precedence;
    return -1;
}

/// location
location::location()
    : column(1)
    , line(1)
    , position(0)
{
}

void location::advanceColumn(size_t count) {
    column += count;
    position += count;
}

void location::advanceLine() {
    line++;
    position++;
    column = 1;
}

static inline bool isDigit(int ch) {
    return unsigned(ch) - '0' < 10;
}

static inline bool isChar(int ch) {
    return (unsigned(ch) | 32) - 'a' < 26;
}

static inline bool isOctal(int ch) {
    return unsigned(ch) - '0' < 8;
}

static inline bool isHex(int ch) {
    return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || isDigit(ch);
}

static inline bool isSpace(int ch) {
    return (ch >= '\t' && ch <= '\r') || ch == ' ';
}

lexer::lexer(const char *string)
    : m_data(string)
    , m_length(0)
    , m_error(0)
{
    if (m_data)
        m_length = strlen(m_data);
}

int lexer::at(int offset) const {
    if (position() + offset < m_length)
        return m_data[position() + offset];
    return 0;
}

void lexer::read(token &out) {
    // Any previous identifier must be freed
    if (out.m_type == kType_identifier)
        free(out.asIdentifier);

    // TODO: Line continuation (backslash `\'.)
    if (position() == m_length) {
        out.m_type = kType_eof;
        return;
    }

    int ch1 = at(1);
    int ch2 = at(2);

    // Lex numerics
    if (isDigit(at()) || (at() == '.' && isDigit(ch1)))
    {
        bool isFloat = false;
        bool isDouble = false;
        bool isUnsigned = false;
        bool isOctalish = false;
        bool isHexish = false;

        if (at() == '0') {
            if (ch1 && (ch1 == 'x' || ch1 == 'X')) {
                isHexish = true;
                m_location.advanceColumn(2);
            } else {
                isOctalish = true;
            }
        }

        vector<char> numeric = readNumeric(isOctalish, isHexish);
        if (position() != m_length && at() == '.') {
            isFloat = true;
            isOctalish = false;
            numeric.push_back('.');
            m_location.advanceColumn();
            vector<char> others = readNumeric(isOctalish, isHexish);
            numeric.reserve(numeric.size() + others.size());
            numeric.insert(numeric.end(), others.begin(), others.end());
        }

        if (position() != m_length && (at() == 'e' || at() == 'E')) {
            ch1 = at(1);
            ch2 = at(2);
            if ((ch1 == '+' || ch1 == '-') && (ch2 >= '0' && ch2 <= '9')) {
                numeric.push_back(ch1);
                numeric.push_back(ch2);
                m_location.advanceColumn(2);
                vector<char> others = readNumeric(isOctalish, isHexish);
                numeric.reserve(numeric.size() + others.size());
                numeric.insert(numeric.end(), others.begin(), others.end());
                isFloat = true;
                isOctalish = false;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
        }

        if (position() != m_length && isChar(at())) {
            ch1 = at(1);
            if (at() == 'f' || at() == 'F') {
                isFloat = true;
                isOctalish = false;
            } else if ((at() == 'l' && ch1 == 'f') || (at() == 'L' && ch1 == 'F')) {
                isFloat = false;
                isDouble = true;
                isOctalish = false;
                m_location.advanceColumn();
            } else if (at() == 'u' || at() == 'U') {
                if (isFloat) {
                    m_error = "invalid use of suffix on literal";
{'fraction_non_alphanumeric': 0.16815721346199106, 'fraction_numerical': 0.010826076723935044, 'mean_word_length': 1.9602925809822362, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5464676', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2799, 'n_words': 681}