<commit_before>/*
 * Use and distribution licensed under the Apache license version 2.
 *
 * See the COPYING file in the root project directory for full text.
 */

#include <vector>

#include "context.h"
#include "error.h"
#include "parser/comment.h"
#include "parser/identifier.h"
#include "parser/lexer.h"
#include "parser/literal.h"
#include "parser/keyword.h"
#include "parser/punctuator.h"
#include "parser/token.h"

namespace sqltoast {

void fill_lexeme(token_t* tok, lexeme_t& lexeme) {
    lexeme.start = tok->lexeme.start;
    lexeme.end = tok->lexeme.end;
}

parse_position_t skip_simple_comments(parse_position_t cursor) {
    parse_position_t start = cursor;
    if (*cursor != '-')
        return start;

    cursor++;
    if (*cursor != '-') {
        return start;
    }

    // The comment content is from the cursor until we find a newline or EOS
    do {
        cursor++;
    } while (*cursor != '\0' && *cursor != '\n');
    return cursor;
}

static size_t NUM_TOKENIZERS = 5;
static tokenize_func_t tokenizers[5] = {
    &token_comment,
    &token_punctuator,
    &token_keyword,
    &token_literal,
    &token_identifier
};

symbol_t lexer_t::peek() const {
    parse_position_t cur = cursor;
    // Advance the lexer's cursor over any whitespace or simple comments
    while (std::isspace(*cur))
        cur++;
    cur = skip_simple_comments(cur);

    for (size_t x = 0; x < NUM_TOKENIZERS; x++) {
        auto tok_res = tokenizers[x](cur);
        if (tok_res.code == TOKEN_FOUND)
            return tok_res.token.symbol;
        if (tok_res.code == TOKEN_NOT_FOUND)
            continue;
        // There was an error in tokenizing... return some error marker?
        return SYMBOL_EOS;
    }
    // No more tokens
    return SYMBOL_EOS;
}

token_t* lexer_t::next() {
    parse_position_t cur = cursor;
    // Advance the lexer's cursor over any whitespace or simple comments
    while (std::isspace(*cur))
        cur++;
    cur = skip_simple_comments(cur);

    for (size_t x = 0; x < NUM_TOKENIZERS; x++) {
        auto tok_res = tokenizers[x](cur);
        if (tok_res.code == TOKEN_FOUND) {
            current_token = tok_res.token;
            cursor = tok_res.token.lexeme.end;
            cursor++;
            return &current_token;
        }
        if (tok_res.code == TOKEN_NOT_FOUND)
            continue;
        // There was an error in tokenizing
        return NULL;
    }
    // No more tokens
    return NULL;
}

} // namespace sqltoast<commit_msg>check for TOKEN_NOT_FOUND before TOKEN_FOUND<commit_after>/*
 * Use and distribution licensed under the Apache license version 2.
 *
 * See the COPYING file in the root project directory for full text.
 */

#include <vector>

#include "context.h"
#include "error.h"
#include "parser/comment.h"
#include "parser/identifier.h"
#include "parser/lexer.h"
#include "parser/literal.h"
#include "parser/keyword.h"
#include "parser/punctuator.h"
#include "parser/token.h"

namespace sqltoast {

void fill_lexeme(token_t* tok, lexeme_t& lexeme) {
    lexeme.start = tok->lexeme.start;
    lexeme.end = tok->lexeme.end;
}

parse_position_t skip_simple_comments(parse_position_t cursor) {
    parse_position_t start = cursor;
    if (*cursor != '-')
        return start;

    cursor++;
    if (*cursor != '-') {
        return start;
    }

    // The comment content is from the cursor until we find a newline or EOS
    do {
        cursor++;
    } while (*cursor != '\0' && *cursor != '\n');
    return cursor;
}

static size_t NUM_TOKENIZERS = 5;
static tokenize_func_t tokenizers[5] = {
    &token_comment,
    &token_punctuator,
    &token_keyword,
    &token_literal,
    &token_identifier
};

symbol_t lexer_t::peek() const {
    parse_position_t cur = cursor;
    // Advance the lexer's cursor over any whitespace or simple comments
    while (std::isspace(*cur))
        cur++;
    cur = skip_simple_comments(cur);

    for (size_t x = 0; x < NUM_TOKENIZERS; x++) {
        auto tok_res = tokenizers[x](cur);
        if (tok_res.code == TOKEN_NOT_FOUND)
            continue;
        if (tok_res.code == TOKEN_FOUND)
            return tok_res.token.symbol;
        // There was an error in tokenizing... return some error marker?
        return SYMBOL_EOS;
    }
    // No more tokens
    return SYMBOL_EOS;
}

token_t* lexer_t::next() {
    parse_position_t cur = cursor;
    // Advance the lexer's cursor over any whitespace or simple comments
    while (std::isspace(*cur))
        cur++;
    cur = skip_simple_comments(cur);

    for (size_t x = 0; x < NUM_TOKENIZERS; x++) {
        auto tok_res = tokenizers[x](cur);
        if (tok_res.code == TOKEN_NOT_FOUND)
            continue;
        if (tok_res.code == TOKEN_FOUND) {
            current_token = tok_res.token;
            cursor = tok_res.token.lexeme.end;
            cursor++;
            return &current_token;
        }
        // There was an error in tokenizing
        return NULL;
    }
    // No more tokens
    return NULL;
}

} // namespace sqltoast
{'fraction_non_alphanumeric': 0.11586701174596854, 'fraction_numerical': 0.002389010551463269, 'mean_word_length': 2.5911365260900645, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1314306', 'n_tokens_mistral': 1687, 'n_tokens_neox': 1634, 'n_words': 443}