<commit_before>#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include "parser.h"
#include "proxy.h"

int main(int argc, char **argv) {
    if (argc != 2) {
        return 1;
    }
    char *buf;
    ssize_t n;
    struct bc_parser parser = { 0 };
    struct proxy_state st = { 0 };
    size_t bufsz;

    sscanf(argv[1], "%zu", &bufsz);
    buf = malloc(bufsz);

    st.argbuf = malloc(4096);
    st.obuf = malloc(4096);
    parser.data = &st;

    parser.on_open = on_open;
    parser.on_close = on_close;
    parser.on_text = on_text;
    parser.on_tag_text = on_tag_text;
    parser.on_arg_end = on_arg_end;
    while ((n = read(STDIN_FILENO, buf, bufsz)) > 0) {
        bc_parse(&parser, buf, n);
        write(STDOUT_FILENO, st.obuf, st.olen);
        st.olen = 0;
    }
}<commit_msg>make test_parser use buffer.h<commit_after>#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include "parser.h"
#include "proxy.h"
#include "buffer.h"

int main(int argc, char **argv) {
    if (argc != 2) {
        return 1;
    }
    char *buf;
    ssize_t n;
    struct bc_parser parser = { 0 };
    struct proxy_state st = { 0 };
    size_t bufsz;

    sscanf(argv[1], "%zu", &bufsz);
    buf = malloc(bufsz);

    st.tmpbuf = buffer_new(4096);
    st.obuf = buffer_new(4096);
    parser.data = &st;

    parser.on_open = on_open;
    parser.on_close = on_close;
    parser.on_text = on_text;
    parser.on_tag_text = on_tag_text;
    parser.on_arg_end = on_arg_end;
    while ((n = read(STDIN_FILENO, buf, bufsz)) > 0) {
        bc_parse(&parser, buf, n);
        write(STDOUT_FILENO, st.obuf->data, st.obuf->len);
        buffer_clear(st.obuf);
    }
}
{'fraction_non_alphanumeric': 0.1521357519016969, 'fraction_numerical': 0.016968987712112346, 'mean_word_length': 2.75, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3339304', 'n_tokens_mistral': 719, 'n_tokens_neox': 676, 'n_words': 146}