<commit_before>#include <sstream>
#include <iomanip>
#include <distributions/io/protobuf.hpp>
#include "protobuf_stream.hpp"
#include "cross_cat.hpp"
#include "infer_grid.hpp"

namespace loom
{

void CrossCat::model_load (const char * filename)
{
    protobuf::CrossCat message;
    protobuf::InFile(filename).read(message);

    schema.clear();
    featureid_to_kindid.clear();
    kinds.clear();

    const size_t kind_count = message.kinds_size();
    kinds.resize(kind_count);
    for (size_t kindid = 0; kindid < kind_count; ++kindid) {
        auto & kind = kinds[kindid];
        const auto & message_kind = message.kinds(kindid);

        kind.featureids.clear();
        std::vector<size_t> ordered_featureids;
        for (size_t i = 0; i < message_kind.featureids_size(); ++i) {
            size_t featureid = message_kind.featureids(i);
            kind.featureids.insert(featureid);
            ordered_featureids.push_back(featureid);
        }

        kind.model.load(message_kind.product_model(), ordered_featureids);
        schema += kind.model.schema;
    }

    distributions::clustering_load(
        feature_clustering,
        message.feature_clustering());

    for (size_t i = 0; i < message.featureid_to_kindid_size(); ++i) {
        featureid_to_kindid.push_back(message.featureid_to_kindid(i));
    }
}

void CrossCat::model_dump (const char * filename) const
{
    protobuf::CrossCat message;

    for (const auto & kind : kinds) {
        auto & message_kind = * message.add_kinds();

        std::vector<size_t> ordered_featureids(
            kind.featureids.begin(),
            kind.featureids.end());
        std::sort(ordered_featureids.begin(), ordered_featureids.end());

        for (size_t i : ordered_featureids) {
            message_kind.add_featureids(i);
        }

        kind.model.dump(* message_kind.mutable_product_model());
    }

    distributions::clustering_dump(
        feature_clustering,
        * message.mutable_feature_clustering());

    for (size_t kindid : featureid_to_kindid) {
        message.add_featureid_to_kindid(kindid);
    }

    protobuf::OutFile(filename).write(message);
}

std::string CrossCat::get_mixture_filename (
        const char * dirname,
        size_t kindid) const
{
    LOOM_ASSERT_LE(kindid, kinds.size());
    std::ostringstream filename;
    filename << dirname << "/mixture." <<<commit_msg>Add hyper_prior serialization, so hyper kernels run<commit_after>#include <sstream>
#include <iomanip>
#include <distributions/io/protobuf.hpp>
#include "protobuf_stream.hpp"
#include "cross_cat.hpp"
#include "infer_grid.hpp"

namespace loom
{

void CrossCat::model_load (const char * filename)
{
    protobuf::CrossCat message;
    protobuf::InFile(filename).read(message);

    schema.clear();
    featureid_to_kindid.clear();
    kinds.clear();

    const size_t kind_count = message.kinds_size();
    kinds.resize(kind_count);
    for (size_t kindid = 0; kindid < kind_count; ++kindid) {
        auto & kind = kinds[kindid];
        const auto & message_kind = message.kinds(kindid);

        kind.featureids.clear();
        std::vector<size_t> ordered_featureids;
        for (size_t i = 0; i < message_kind.featureids_size(); ++i) {
            size_t featureid = message_kind.featureids(i);
            kind.featureids.insert(featureid);
            ordered_featureids.push_back(featureid);
        }

        kind.model.load(message_kind.product_model(), ordered_featureids);
        schema += kind.model.schema;
    }

    distributions::clustering_load(
        feature_clustering,
        message.feature_clustering());

    for (size_t i = 0; i < message.featureid_to_kindid_size(); ++i) {
        featureid_to_kindid.push_back(message.featureid_to_kindid(i));
    }

    hyper_prior = message.hyper_prior();
}

void CrossCat::model_dump (const char * filename) const
{
    protobuf::CrossCat message;

    for (const auto & kind : kinds) {
        auto & message_kind = * message.add_kinds();

        std::vector<size_t> ordered_featureids(
            kind.featureids.begin(),
            kind.featureids.end());
        std::sort(ordered_featureids.begin(), ordered_featureids.end());

        for (size_t i : ordered_featureids) {
            message_kind.add_featureids(i);
        }

        kind.model.dump(* message_kind.mutable_product_model());
    }

    distributions::clustering_dump(
        feature_clustering,
        * message.mutable_feature_clustering());

    for (size_t kindid : featureid_to_kindid) {
        message.add_featureid_to_kindid(kindid);
    }

    * message.mutable_hyper_prior() = hyper_prior;

    protobuf::OutFile(filename).write(message);
}

std::string CrossCat::get_mixture_filename (
        const char * dirname,
        size_t kindid) const
{
    LOOM_ASSERT_LE(kindid, kinds.size());
    std::ostringstream filename;
    filename << dirname << "/mixture." <<
{'fraction_non_alphanumeric': 0.12189156135344476, 'fraction_numerical': 0.0012229922543823889, 'mean_word_length': 3.2011986301369864, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080629', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1531, 'n_words': 264}