<commit_before>#ifndef LEXER_H_
#define LEXER_H_

typedef struct Token {
    int type;
    char * value;
    char quote;
} token;


struct Token * append(struct Token * tok_arr, int ntokens, int type, char quote,
    char * val);
char * append_to_buffer(char * buffer, char chr);
char * copy_str(char * from);
void free_token_array(struct Token * token_array, int size);
void print_token_array(struct Token * token_array, int size);

#endif<commit_msg>Fit function prototype onto one line<commit_after>#ifndef LEXER_H_
#define LEXER_H_

typedef struct Token {
    int type;
    char * value;
    char quote;
} token;


struct Token * append(struct Token * toks, int ntoks, int t, char q, char * v);
char * append_to_buffer(char * buffer, char chr);
char * copy_str(char * from);
void free_token_array(struct Token * token_array, int size);
void print_token_array(struct Token * token_array, int size);

#endif
{'fraction_non_alphanumeric': 0.0968096809680968, 'fraction_numerical': 0.0, 'mean_word_length': 4.6875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898393', 'n_tokens_mistral': 307, 'n_tokens_neox': 298, 'n_words': 100}