<commit_before>#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-token.h"

%%{
    machine set;
    write data;
}%%

bel_token_list* bel_tokenize_term(char* line) {
    int                  cs;
    char                 *p;
    char                 *pe;
    char                 *ts;
    char                 *te;
    int                  act;
    char                 *eof;
    int                  token_i;
    bel_token_list       *token_list;
    bel_token            *tokens;
    bel_token            *new_token;

    p                  = line;
    pe                 = line + strlen(line);
    eof                = pe;
    token_i            = 0;
    token_list         = bel_new_token_list(256);
    tokens             = token_list->tokens;

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"';

        action IDENT   {
            new_token = bel_new_token(IDENT, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action STRING  {
            new_token = bel_new_token(STRING, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action O_PAREN {
            new_token = bel_new_token(O_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action C_PAREN {
            new_token = bel_new_token(C_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COLON   {
            new_token = bel_new_token(COLON, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COMMA   {
            new_token = bel_new_token(COMMA, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action SPACES  {
            new_token = bel_new_token(SPACES, line, ts, te);
            tokens[token_i++] = *new_token;
        }<commit_msg>tokenizing STRING can leave off last double quote<commit_after>#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-token.h"

%%{
    machine set;
    write data;
}%%

bel_token_list* bel_tokenize_term(char* line) {
    int                  cs;
    char                 *p;
    char                 *pe;
    char                 *ts;
    char                 *te;
    char                 *eof;
    int                  token_i;
    bel_token_list       *token_list;
    bel_token            *tokens;
    bel_token            *new_token;

    p                  = line;
    pe                 = line + strlen(line);
    eof                = pe;
    token_i            = 0;
    token_list         = bel_new_token_list(256);
    tokens             = token_list->tokens;

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"'?;

        action IDENT   {
            new_token = bel_new_token(IDENT, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action STRING  {
            new_token = bel_new_token(STRING, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action O_PAREN {
            new_token = bel_new_token(O_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action C_PAREN {
            new_token = bel_new_token(C_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COLON   {
            new_token = bel_new_token(COLON, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COMMA   {
            new_token = bel_new_token(COMMA, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action SPACES  {
            new_token = bel_new_token(SPACES, line, ts, te);
            tokens[token_i++] = *new_token;
        }
{'fraction_non_alphanumeric': 0.12040969273045216, 'fraction_numerical': 0.0029977516862353237, 'mean_word_length': 1.2456533931575995, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4398803', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1280, 'n_words': 247}