<commit_before>// Test harness for the HtmlParser class.
#include <iostream>
#include <cstdlib>
#include "FileUtil.h"
#include "HtmlParser.h"
#include "util.h"


void Usage() {
    std::cerr << "Usage: " << ::progname << " html_filename\n";
    std::exit(EXIT_FAILURE);
}


class Parser: public HtmlParser {
public:
    Parser(const std::string &document): HtmlParser(document) { }
    virtual void notify(const HtmlParser::Chunk &chunk);
};


void Parser::notify(const HtmlParser::Chunk &chunk) {
    std::cout << chunk.toString() << '\n';
}


int main(int argc, char *argv[]) {
    ::progname = argv[1];
    if (argc != 2)
        Usage();<commit_msg>Fixed an invalid default argument bug.<commit_after>// Test harness for the HtmlParser class.
#include <iostream>
#include <cstdlib>
#include "FileUtil.h"
#include "HtmlParser.h"
#include "util.h"


void Usage() {
    std::cerr << "Usage: " << ::progname << " html_filename\n";
    std::exit(EXIT_FAILURE);
}


class Parser: public HtmlParser {
public:
    Parser(const std::string &document)
        : HtmlParser(document,
                     HtmlParser::EVERYTHING & ~(HtmlParser::WORD | HtmlParser::PUNCTUATION | HtmlParser::WHITESPACE)) { }
    virtual void notify(const HtmlParser::Chunk &chunk);
};


void Parser::notify(const HtmlParser::Chunk &chunk) {
    std::cout << chunk.toString() << '\n';
}


int main(int argc, char *argv[]) {
    ::progname = argv[1];
    if (argc != 2)
        Usage();
{'fraction_non_alphanumeric': 0.16598079561042525, 'fraction_numerical': 0.0027434842249657062, 'mean_word_length': 3.929054054054054, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754520', 'n_tokens_mistral': 508, 'n_tokens_neox': 486, 'n_words': 120}