<commit_before>// sota.cpp : Defines the entry point for the console application.
//
#include <array>
#include <string>
#include <iostream>
#include <exception>

#include "z2h/parser.hpp"
#include "z2h/symbol.hpp"

#include "ast.h"
#include "parser.h"
#include "bindpower.h"

#include "tclap/CmdLine.h"

using namespace sota;

int main(int argc, char **argv) {
    try {
        TCLAP::CmdLine cmdline("Sota Description", ' ', "0.1");
        TCLAP::SwitchArg tokenize("t", "tokenize", "Tokenize the code", cmdline, false);
        TCLAP::ValueArg<std::string> code("c", "code", "Code to evaluate", true, "", "string");
        TCLAP::UnlabeledValueArg<std::string> file("file", "Sota file with main definined", true, "",  "file");
        cmdline.xorAdd(code, file);

        cmdline.parse(argc, argv);

        std::string source = code.getValue();
        std::string filename = file.getValue();

        auto parser = SotaParser();

        if (tokenize.getValue()) {
            std::vector<SotaToken *> tokens;
            if (!filename.empty()) {
                tokens = parser.TokenizeFile(filename);
            }
            else if (!source.empty()) {
                tokens = parser.Tokenize(source);
            }
            for (auto token : tokens)
                std::cout << *token << std::endl;
            return 0;
        }

        Ast *ast = nullptr;
        if (!filename.empty()) {
            ast = parser.ParseFile(filename);
            std::cout << filename << ":" << std::endl;
        }
        else if (!source.empty()) {
            ast = parser.Parse(source);
            std::cout << source << ":" << std::endl;
        }
        std::cout << ast->Print() << std::endl;
    }
    catch (TCLAP::ArgException &e) {
        std::cerr << "error: " << e.error() << " for arg " << e.argId() << std::endl;
        return 1;
    }
    return 0;
}<commit_msg>removed : after code and file in print... -sai<commit_after>// sota.cpp : Defines the entry point for the console application.
//
#include <array>
#include <string>
#include <iostream>
#include <exception>

#include "z2h/parser.hpp"
#include "z2h/symbol.hpp"

#include "ast.h"
#include "parser.h"
#include "bindpower.h"

#include "tclap/CmdLine.h"

using namespace sota;

int main(int argc, char **argv) {
    try {
        TCLAP::CmdLine cmdline("Sota Description", ' ', "0.1");
        TCLAP::SwitchArg tokenize("t", "tokenize", "Tokenize the code", cmdline, false);
        TCLAP::ValueArg<std::string> code("c", "code", "Code to evaluate", true, "", "string");
        TCLAP::UnlabeledValueArg<std::string> file("file", "Sota file with main definined", true, "",  "file");
        cmdline.xorAdd(code, file);

        cmdline.parse(argc, argv);

        std::string source = code.getValue();
        std::string filename = file.getValue();

        auto parser = SotaParser();

        if (tokenize.getValue()) {
            std::vector<SotaToken *> tokens;
            if (!filename.empty()) {
                tokens = parser.TokenizeFile(filename);
            }
            else if (!source.empty()) {
                tokens = parser.Tokenize(source);
            }
            for (auto token : tokens)
                std::cout << *token << std::endl;
            return 0;
        }

        Ast *ast = nullptr;
        if (!filename.empty()) {
            ast = parser.ParseFile(filename);
            std::cout << filename << std::endl;
        }
        else if (!source.empty()) {
            ast = parser.Parse(source);
            std::cout << source << std::endl;
        }
        std::cout << ast->Print() << std::endl;
    }
    catch (TCLAP::ArgException &e) {
        std::cerr << "error: " << e.error() << " for arg " << e.argId() << std::endl;
        return 1;
    }
    return 0;
}
{'fraction_non_alphanumeric': 0.16679841897233202, 'fraction_numerical': 0.003689064558629776, 'mean_word_length': 2.225148683092608, 'pattern_counts': {'":': 2, '<': 73, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '172365', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1155, 'n_words': 283}