<commit_before>#include "support.h"

#include <fstream>
#include <iso646.h>
#include <sstream>

#include <sha256.h>

#include "lexer.h"
#include "parser.h"
#include "compiler.h"

bool CompilerSupport::loadBytecode(const std::string &name, Bytecode &object)
{
    std::pair<std::string, std::string> names = findModule(name);
    if (names.first.empty() && names.second.empty()) {
        return false;
    }

    std::ifstream obj(names.second, std::ios::binary);
    std::ifstream src(names.first);

    std::string source;
    if (src.good()) {
        std::stringstream buf;
        buf << src.rdbuf();
        source = buf.str();
    }

    if (obj.good() && not source.empty()) {
        SHA256 sha256;
        sha256(source);
        unsigned char h[SHA256::HashBytes];
        sha256.getHash(h);
        std::string hash = std::string(h, h+sizeof(h));

        std::stringstream buf;
        buf << obj.rdbuf();
        std::vector<unsigned char> bytecode;
        std::string s = buf.str();
        std::copy(s.begin(), s.end(), std::back_inserter(bytecode));
        object = Bytecode(bytecode);

        // TODO: skipping the following check causes modules to always be rebuilt
        //if (object.source_hash == hash) {
        //    return true;
        //}
    }

    if (not source.empty()) {
        obj.close();
        const std::string objname = names.first + "x";
        remove(objname.c_str());
        {
            auto tokens = tokenize(names.first, source);
            auto parsetree = parse(tokens);
            auto ast = analyze(this, parsetree);
            auto bytecode = compile(ast, nullptr);
            std::ofstream outf(objname, std::ios::binary);
            outf.write(reinterpret_cast<const std::ofstream::char_type *>(bytecode.data()), bytecode.size());<commit_msg>Always rebuild all modules, avoiding reading the old bytecode for now<commit_after>#include "support.h"

#include <fstream>
#include <iso646.h>
#include <sstream>

#include <sha256.h>

#include "lexer.h"
#include "parser.h"
#include "compiler.h"

bool CompilerSupport::loadBytecode(const std::string &name, Bytecode &object)
{
    std::pair<std::string, std::string> names = findModule(name);
    if (names.first.empty() && names.second.empty()) {
        return false;
    }

    std::ifstream obj(names.second, std::ios::binary);
    std::ifstream src(names.first);

    std::string source;
    if (src.good()) {
        std::stringstream buf;
        buf << src.rdbuf();
        source = buf.str();
    }

    // TODO: skipping the following check causes modules to always be rebuilt
    //if (obj.good() && not source.empty()) {
    //    SHA256 sha256;
    //    sha256(source);
    //    unsigned char h[SHA256::HashBytes];
    //    sha256.getHash(h);
    //    std::string hash = std::string(h, h+sizeof(h));
    //
    //    std::stringstream buf;
    //    buf << obj.rdbuf();
    //    std::vector<unsigned char> bytecode;
    //    std::string s = buf.str();
    //    std::copy(s.begin(), s.end(), std::back_inserter(bytecode));
    //    object = Bytecode(bytecode);
    //
    //    if (object.source_hash == hash) {
    //        return true;
    //    }
    //}

    if (not source.empty()) {
        obj.close();
        const std::string objname = names.first + "x";
        remove(objname.c_str());
        {
            auto tokens = tokenize(names.first, source);
            auto parsetree = parse(tokens);
            auto ast = analyze(this, parsetree);
            auto bytecode = compile(ast, nullptr);
            std::ofstream outf(objname, std::ios::binary);
            outf.write(reinterpret_cast<const std::ofstream::char_type *>(bytecode.data()), bytecode.size());
{'fraction_non_alphanumeric': 0.16109012412304372, 'fraction_numerical': 0.011332973556395036, 'mean_word_length': 2.520417853751187, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7524390', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1160, 'n_words': 257}