<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <err.h>
#include <unistd.h>

#include "parse.h"

#include "gram.h"

#define End (-1)

char *filename;
int line;
Tok *curtok;

/* the file contents are stored globally */
static int   fidx;
static int   fbufsz;
static char *fbuf;

static int peekn(int n)
{
    if (fidx + n >= fbufsz)
        return End;
    else
        return fbuf[fidx + n];
}

static int peek(void)
{
    return peekn(0);
}

static int next(void)
{
    int c;

    c = peek();
    fidx++;
    return c;
}

static void unget(void)
{
    fidx--;
    assert(fidx >= 0);
}

/*
 * Consumes the character iff
 * the character is equal to 'c'.
 * returns true if there was a match,
 * false otherwise.
 */
static int match(char c)
{
    if (peek() == c) {
        next();
        return 1;
    } else {
        return 0;
    }
}

static Tok *mktok(int tt)
{
    Tok *t;

    t = zalloc(sizeof(Tok));
    t->type = tt;
    t->line = line;
    return t;
}

static int identchar(int c)
{
    return isalnum(c) || c == '_';
}

static void eatcomment(void)
{
    int depth;
    int startln;
    int c;

    depth = 0;
    startln = line;
    while (1) {
        c = next();
        switch (c) {
            /* enter level of nesting */
            case '/':
                if (match('*'))
                    depth++;
                break;
            /* leave level of nesting */
            case '*':
                if (match('/'))
                    depth--;
                break;
            /* have to keep line numbers synced */
            case '\n':
                line++;
                break;
            case End:
                fatal(line, "File ended within comment starting at line %d", startln);
                break;
        }
        if (depth == 0)
            break;
    }
}

/*
 * Consumes all forms of whitespace,
 * including comments. If we are in a
 * state where we should ignore newlines,
 * we also consume '\n'. ';' is still
 * accepted as a line ending.
 */
static void eatspace(void)
{
    int c;
    int ignorenl;

    ignorenl = 0;
    while (1) {
        c = peek();
        if (!ignorenl && c == '\n') {
            ignorenl = 0;
            break;
        } else if (c == '\\') {
            ignorenl = 1;
            next();
        } else if (isspace(c) || (ignorenl && c == '\n')) {
            next();
        } else if (c == '/' && peekn(1) == '*') {
            eatcomment();
        } else {
            break;
        }
    }
}

/*
 * Decides if an identifier is a
 * keyword or not. Returns the
 * token type to use for the
 * identifier.
 */
static int kwd(char *s)
{
    static const struct {char* kw; int tt;} kwmap[] = {
        {"break",       Tbreak},
        {"castto",      Tcast},
        {"const",       Tconst},<commit_msg>Track line number correctly in tokenizer<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <err.h>
#include <unistd.h>

#include "parse.h"

#include "gram.h"

#define End (-1)

char *filename;
int line;
Tok *curtok;

/* the file contents are stored globally */
static int   fidx;
static int   fbufsz;
static char *fbuf;

static int peekn(int n)
{
    if (fidx + n >= fbufsz)
        return End;
    else
        return fbuf[fidx + n];
}

static int peek(void)
{
    return peekn(0);
}

static int next(void)
{
    int c;

    c = peek();
    fidx++;
    return c;
}

static void unget(void)
{
    fidx--;
    assert(fidx >= 0);
}

/*
 * Consumes the character iff
 * the character is equal to 'c'.
 * returns true if there was a match,
 * false otherwise.
 */
static int match(char c)
{
    if (peek() == c) {
        next();
        return 1;
    } else {
        return 0;
    }
}

static Tok *mktok(int tt)
{
    Tok *t;

    t = zalloc(sizeof(Tok));
    t->type = tt;
    t->line = line;
    return t;
}

static int identchar(int c)
{
    return isalnum(c) || c == '_';
}

static void eatcomment(void)
{
    int depth;
    int startln;
    int c;

    depth = 0;
    startln = line;
    while (1) {
        c = next();
        switch (c) {
            /* enter level of nesting */
            case '/':
                if (match('*'))
                    depth++;
                break;
            /* leave level of nesting */
            case '*':
                if (match('/'))
                    depth--;
                break;
            /* have to keep line numbers synced */
            case '\n':
                line++;
                break;
            case End:
                fatal(line, "File ended within comment starting at line %d", startln);
                break;
        }
        if (depth == 0)
            break;
    }
}

/*
 * Consumes all forms of whitespace,
 * including comments. If we are in a
 * state where we should ignore newlines,
 * we also consume '\n'. ';' is still
 * accepted as a line ending.
 */
static void eatspace(void)
{
    int c;
    int ignorenl;

    ignorenl = 0;
    while (1) {
        c = peek();
        if (!ignorenl && c == '\n') {
            ignorenl = 0;
            break;
        } else if (c == '\\') {
            ignorenl = 1;
            next();
        } else if (ignorenl && c == '\n') {
            next();
            line++;
            ignorenl = 0;
        } else if (isspace(c)) {
            next();
        } else if (c == '/' && peekn(1) == '*') {
            eatcomment();
        } else {
            break;
        }
    }
}

/*
 * Decides if an identifier is a
 * keyword or not. Returns the
 * token type to use for the
 * identifier.
 */
static int kwd(char *s)
{
    static const struct {char* kw; int tt;} kwmap[] = {
        {"break",       Tbreak},
        {"castto",      Tcast},
        {"const",       Tconst},
{'fraction_non_alphanumeric': 0.1434710743801653, 'fraction_numerical': 0.004462809917355372, 'mean_word_length': 1.5792838874680306, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7074824', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1968, 'n_words': 628}