<commit_before>#include "common.h"

#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "fail.h"
#include "lexer.h"


#define endof(p) ((char *)p + sizeof(*p))

#define BUFCAP 10


/*!max:re2c*/
enum { MAXFILL = YYMAXFILL };
#undef YYMAXFILL


struct lexer {
    int fd;
    char buf[BUFCAP + MAXFILL + 1];
    char* cursor;
    char* limit;<commit_msg>Increase buffer capacity to a reasonable size<commit_after>#include "common.h"

#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "fail.h"
#include "lexer.h"


#define endof(p) ((char *)p + sizeof(*p))

#define BUFCAP 128


/*!max:re2c*/
enum { MAXFILL = YYMAXFILL };
#undef YYMAXFILL


struct lexer {
    int fd;
    char buf[BUFCAP + MAXFILL + 1];
    char* cursor;
    char* limit;
{'fraction_non_alphanumeric': 0.16573348264277715, 'fraction_numerical': 0.010078387458006719, 'mean_word_length': 3.994413407821229, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7516439', 'n_tokens_mistral': 388, 'n_tokens_neox': 366, 'n_words': 93}