<commit_before>#include <fstream>
#include <iostream>

#include "parsing/parser.h"
#include "vm.h"
#include "ast/ast.h"
#include "ast/pretty-printer.h"
#include "ast/dot-generator.h"
#include "bytecode/compiler.h"
#include "argparser.h"


void print_bytecode(const pl0::bytecode &code) {
    for (size_t i = 0; i < code.size(); i++) {
        std::cout << i << '\t' << *code[i].op << '\t'
            << code[i].level << '\t' << code[i].address << '\n';
    }
}

struct options {
    bool show_bytecode = false;
    bool compile_only = false;
    bool show_ast = false;
    std::string output_graph_file = "";
    std::string input_file = "";
};

options parse_args(int argc, const char *argv[]) {
    try {
        options option;
        std::vector<std::string> rest;
        pl0::argument_parser<options> parser{"The PL/0 compiler"};
        parser.flags({"--show-bytecode", "-s"}, &options::show_bytecode);
        parser.flags({"--compile-only", "-c"}, &options::compile_only);
        parser.flags({"--show-ast", "-t"}, &options::show_ast);
        parser.store<std::initializer_list<const char *>>({"--plot-tree", "-t"}, &options::output_graph_file);
        parser.parse(argc, argv, option, rest);

        if (rest.empty())
            parser.show_help();

        option.input_file = rest[0];
        return option;
    } catch (pl0::basic_error &error) {
        std::cout << "Error: " << error.what() << '\n';
        exit(EXIT_FAILURE);
    }
}

int main(int argc, const char* argv[]) {
    options option = parse_args(argc, argv);

    std::ifstream fin(option.input_file);
    if (fin.fail()) {
        std::cerr << "Error: failed to open file: \"" << option.input_file << "\"\n";
        return 1;
    }

    pl0::lexer lex(fin);
    pl0::parser parser(lex);
    pl0::ast::block *program = nullptr;

    try {
        program = parser.program();
    } catch (pl0::general_error &error) {
        pl0::location loc = lex.current_location();
        std::cout << "Error(" << loc.to_string() << "): " << error.what() << '\n';
        return EXIT_FAILURE;
    }


    pl0::code::compiler compiler{};
    compiler.generate(program);

    if (!option.output_graph_file.empty()) {
        pl0::ast::dot_generator plotter;
        plotter.generate(program);
        plotter.save_to_file(option.output_graph_file.c_str());
    }

    if (option.show_ast) {<commit_msg>feat: add flag to show tokens<commit_after>#include <fstream>
#include <iostream>

#include "parsing/parser.h"
#include "vm.h"
#include "ast/ast.h"
#include "ast/pretty-printer.h"
#include "ast/dot-generator.h"
#include "bytecode/compiler.h"
#include "argparser.h"


void print_bytecode(const pl0::bytecode &code) {
    for (size_t i = 0; i < code.size(); i++) {
        std::cout << i << '\t' << *code[i].op << '\t'
            << code[i].level << '\t' << code[i].address << '\n';
    }
}

struct options {
    bool show_bytecode = false;
    bool show_tokens = false;
    bool compile_only = false;
    bool show_ast = false;
    std::string output_graph_file = "";
    std::string input_file = "";
};

options parse_args(int argc, const char *argv[]) {
    try {
        options option;
        std::vector<std::string> rest;
        pl0::argument_parser<options> parser{"The PL/0 compiler"};
        parser.flags({"--show-bytecode", "-s"}, &options::show_bytecode);
        parser.flags({"--show-tokens", "-l"}, &options::show_tokens);
        parser.flags({"--compile-only", "-c"}, &options::compile_only);
        parser.flags({"--show-ast", "-t"}, &options::show_ast);
        parser.store<std::initializer_list<const char *>>({"--plot-tree", "-t"}, &options::output_graph_file);
        parser.parse(argc, argv, option, rest);

        if (rest.empty())
            parser.show_help();

        option.input_file = rest[0];
        return option;
    } catch (pl0::basic_error &error) {
        std::cout << "Error: " << error.what() << '\n';
        exit(EXIT_FAILURE);
    }
}

[[noreturn]] void print_tokens(pl0::lexer &lex) {
    while (true) {
        auto token = lex.peek();
        std::cout << lex.current_location().to_string() << '\t' << *token << '\t' << lex.get_literal() << '\n';
        if (token == pl0::token::EOS)
            break;
        lex.advance();
    }
    exit(EXIT_SUCCESS);
}

int main(int argc, const char* argv[]) {
    options option = parse_args(argc, argv);

    std::ifstream fin(option.input_file);
    if (fin.fail()) {
        std::cerr << "Error: failed to open file: \"" << option.input_file << "\"\n";
        return 1;
    }

    pl0::lexer lex(fin);

    if (option.show_tokens)
        print_tokens(lex);

    pl0::parser parser(lex);
    pl0::ast::block *program = nullptr;

    try {
        program = parser.program();
    } catch (pl0::general_error &error) {
        pl0::location loc = lex.current_location();
        std::cout << "Error(" << loc.to_string() << "): " << error.what() << '\n';
        return EXIT_FAILURE;
    }


    pl0::code::compiler compiler{};
    compiler.generate(program);

    if (!option.output_graph_file.empty()) {
        pl0::ast::dot_generator plotter;
        plotter.generate(program);
        plotter.save_to_file(option.output_graph_file.c_str());
    }

    if (option.show_ast) {
{'fraction_non_alphanumeric': 0.18735719725818736, 'fraction_numerical': 0.0057121096725057125, 'mean_word_length': 2.8539985326485695, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893422', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1745, 'n_words': 354}