<commit_before>#include <ctype.h>
#include <setjmp.h>
#include <stdarg.h>
#include <string.h>
#include <stdbool.h>
#include <stdnoreturn.h>
#include <assert.h>
#include <errno.h>
#include <math.h>

#include "vec.h"
#include "token.h"
#include "test.h"
#include "util.h"
#include "lex.h"
#include "log.h"

enum {
        MAX_OP_LEN   = 8,
};

static char const *filename;
static struct location Start;

static jmp_buf jb;

static LexState state;
static vec(LexState) states;

#define SRC state.loc.s
#define END state.end

inline static unsigned char
C(int n)
{
        if (SRC + n < END)
                return SRC[n];
        return '\0';
}

static char const *opchars = "/=<~|!@%^&*-+>?.$";

noreturn static void
error(char const *fmt, ...)
{
        va_list ap;
        va_start(ap, fmt);

        int sz = ERR_SIZE - 1;
        int n = snprintf(
                ERR,
                sz,
                "%s%sSyntaxError%s%s %s%s%s:%s%d%s:%s%d%s: ",
                TERM(1),
                TERM(31),
                TERM(22),
                TERM(39),
                TERM(34),
                filename,
                TERM(39),
                TERM(33),
                state.loc.line + 1,
                TERM(39),
                TERM(33),
                state.loc.col + 1,
                TERM(39)
        );
        n += vsnprintf(ERR + n, sz - n, fmt, ap);
        va_end(ap);

        char const *prefix = state.loc.s;
        while (prefix[-1] != '\n' && prefix[-1] != '\0')
                --prefix;

        int before = state.loc.s - prefix;
        int after = strcspn(state.loc.s + 1, "\n");

        n += snprintf(
                ERR + n,
                sz - n,
                "\n\n\tnear: %.*s%s%s%.1s%s%s%.*s\n",
                before,
                prefix,
                TERM(1),
                TERM(31),
                state.loc.s,
                TERM(22),
                TERM(39),
                after,
                state.loc.s + 1<commit_msg>fix off-by-one bug in lexer error reporting<commit_after>#include <ctype.h>
#include <setjmp.h>
#include <stdarg.h>
#include <string.h>
#include <stdbool.h>
#include <stdnoreturn.h>
#include <assert.h>
#include <errno.h>
#include <math.h>

#include "vec.h"
#include "token.h"
#include "test.h"
#include "util.h"
#include "lex.h"
#include "log.h"

enum {
        MAX_OP_LEN   = 8,
};

static char const *filename;
static struct location Start;

static jmp_buf jb;

static LexState state;
static vec(LexState) states;

#define SRC state.loc.s
#define END state.end

inline static unsigned char
C(int n)
{
        if (SRC + n < END)
                return SRC[n];
        return '\0';
}

static char const *opchars = "/=<~|!@%^&*-+>?.$";

noreturn static void
error(char const *fmt, ...)
{
        va_list ap;
        va_start(ap, fmt);

        int sz = ERR_SIZE - 1;
        int n = snprintf(
                ERR,
                sz,
                "%s%sSyntaxError%s%s %s%s%s:%s%d%s:%s%d%s: ",
                TERM(1),
                TERM(31),
                TERM(22),
                TERM(39),
                TERM(34),
                filename,
                TERM(39),
                TERM(33),
                state.loc.line + 1,
                TERM(39),
                TERM(33),
                state.loc.col + 1,
                TERM(39)
        );
        n += vsnprintf(ERR + n, sz - n, fmt, ap);
        va_end(ap);

        char const *prefix = state.loc.s;
        while (prefix[-1] != '\n' && prefix[-1] != '\0')
                --prefix;

        int before = state.loc.s - prefix;
        int after = (state.loc.s[0] == '\0') ? 0 : strcspn(state.loc.s + 1, "\n");

        n += snprintf(
                ERR + n,
                sz - n,
                "\n\n\tnear: %.*s%s%s%.1s%s%s%.*s\n",
                before,
                prefix,
                TERM(1),
                TERM(31),
                state.loc.s,
                TERM(22),
                TERM(39),
                after,
                state.loc.s + 1
{'fraction_non_alphanumeric': 0.1536361379995036, 'fraction_numerical': 0.0191114420451725, 'mean_word_length': 1.3214285714285714, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '553243', 'n_tokens_mistral': 1516, 'n_tokens_neox': 1377, 'n_words': 318}