<commit_before>#include "include/amici_hdf5.h"
#include "include/amici.h"
#include "include/amici_model.h"
#include "include/edata.h"
#include "include/rdata.h"
#include <amici_solver.h>
#include <amici_exception.h>

#include <hdf5_hl.h>

#include <cassert>
#include <cstring>
#include <algorithm>
#ifdef AMI_HDF5_H_DEBUG
#ifndef __APPLE__
#include <cexecinfo>
#else
#include <execinfo.h>
#endif
#endif
#include <unistd.h>
#include <cmath>


namespace amici {
namespace hdf5 {

/**
 * @brief assertMeasurementDimensionsCompatible
 * @param m
 * @param n
 * @param model
 */
void assertMeasurementDimensionsCompatible(hsize_t m, hsize_t n, Model const& model) {
    // if this is rank 1, n and m can be swapped
    if (n == 1) {
        assert(n == (unsigned)model.nt() || n == (unsigned)model.nytrue);
        assert(m == (unsigned)model.nytrue || m == (unsigned)model.nt());
        assert(m * n == (unsigned)model.nytrue * model.nt());
    } else {
        assert(n == (unsigned)model.nt());
        assert(m == (unsigned)model.nytrue);
    }
}


/**
 * @brief assertEventDimensionsCompatible
 * @param m
 * @param n
 * @param model
 */
void assertEventDimensionsCompatible(hsize_t m, hsize_t n, Model const& model) {
    // if this is rank 1, n and m can be swapped
    if (n == 1) {
        assert(n == (unsigned)model.nMaxEvent() ||
               n == (unsigned)model.nztrue);
        assert(m == (unsigned)model.nztrue ||
               m == (unsigned)model.nMaxEvent());
        assert(m * n == (unsigned)model.nytrue * model.nMaxEvent());
    } else {
        assert(n == (unsigned)model.nMaxEvent());
        assert(m == (unsigned)model.nztrue);
    }
}


void createGroup(H5::H5File &file,
                 std::string const& groupPath,
                 bool recursively) {

    hid_t groupCreationPropertyList = H5P_DEFAULT;

    if (recursively) {
        groupCreationPropertyList = H5Pcreate(H5P_LINK_CREATE);
        H5Pset_create_intermediate_group(groupCreationPropertyList, 1);
    }

    hid_t group = H5Gcreate(file.getId(), groupPath.c_str(),
                            groupCreationPropertyList,
                            H5P_DEFAULT, H5P_DEFAULT);
    H5Pclose(groupCreationPropertyList);

    if (group < 0)
        throw(AmiException("Failed to create group in hdf5CreateGroup: %s", groupPath.c_str()));
    H5Gclose(group);
}

std::unique_ptr<ExpData> readSimulationExpData(std::string const& hdf5Filename,
                                               std::string const& hdf5Root,
                                               Model const& model) {
    H5::H5File file(hdf5Filename, H5F_ACC_RDONLY);

    hsize_t m, n;

    auto edata = std::unique_ptr<ExpData>(new ExpData(model));

    auto my = getDoubleArrayAttribute2D(file, hdf5Root, "Y", m, n);
    if(m * n > 0) {
        assertMeasurementDimensionsCompatible(m, n, model);
        edata->my = my;
    }

    auto sigmay = getDoubleArrayAttribute2D(file, hdf5Root, "Sigma_Y", m, n);
    if(m * n > 0) {
        assertMeasurementDimensionsCompatible(m, n, model);
        edata->sigmay = sigmay;
    }

    if (model.nz) {
        edata->mz = getDoubleArrayAttribute2D(file, hdf5Root, "Z", m, n);
        assertEventDimensionsCompatible(m, n, model);

        edata->sigmaz = getDoubleArrayAttribute2D(file, hdf5Root, "Sigma_Z", m, n);
        assertEventDimensionsCompatible(m, n, model);
    }

    return edata;
}

void writeReturnData(ReturnData const& rdata,
                     std::string const& hdf5Filename,
                     std::string const& hdf5Location) {
    auto file = createOrOpenForWriting(hdf5Filename);

    if(!locationExists(file, hdf5Location))
        createGroup(file, hdf5Location);

    if (rdata.ts)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "t",
                                 rdata.ts, rdata.nt);

    if (rdata.xdot)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "xdot", rdata.xdot,
                                 rdata.nx);

    if (rdata.llh)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "llh", rdata.llh, 1);
    
    if (rdata.status)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "status", rdata.status, 1);

    if (rdata.sllh)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "sllh", rdata.sllh,
                                 rdata.nplist);

    // are double, but should write as int:
    if (rdata.numsteps)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(), "numsteps",
                                  rdata.numsteps, rdata.nt);

    if (rdata.numrhsevals)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(), "numrhsevals",
                                  rdata.numrhsevals, rdata.nt);

    if (rdata.numerrtestfails)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(),
                                  "numerrtestfails",
                                  rdata.numerrtestfails, rdata.nt);

    if (rdata.numnonlinsolvconvfails)
        setAttributeIntFromDouble(
                    file.getId(), hdf5Location.c_str(), "numnonlinsolvconvfails",
                    rdata.numnonlinsolvconvfails, rdata.nt);

    if (rdata.order)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(), "order",
                                  rdata.order, rdata.nt);

    if (rdata.numstepsB)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(), "numstepsB",
                                  rdata.numstepsB, rdata.nt);

    if (rdata.numrhsevalsB)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(), "numrhsevalsB",
                                  rdata.numrhsevalsB, rdata.nt);

    if (rdata.numerrtestfailsB)
        setAttributeIntFromDouble(file.getId(), hdf5Location.c_str(),
                                  "numerrtestfailsB",
                                  rdata.numerrtestfailsB, rdata.nt);

    if (rdata.numnonlinsolvconvfailsB)
        setAttributeIntFromDouble(
                    file.getId(), hdf5Location.c_str(), "numnonlinsolvconvfailsB",
                    rdata.numnonlinsolvconvfailsB, rdata.nt);

    auto group = file.openGroup(hdf5Location);<commit_msg>How could my compiler not complain about that?<commit_after>#include "include/amici_hdf5.h"
#include "include/amici.h"
#include "include/amici_model.h"
#include "include/edata.h"
#include "include/rdata.h"
#include <amici_solver.h>
#include <amici_exception.h>

#include <hdf5_hl.h>

#include <cassert>
#include <cstring>
#include <algorithm>
#ifdef AMI_HDF5_H_DEBUG
#ifndef __APPLE__
#include <cexecinfo>
#else
#include <execinfo.h>
#endif
#endif
#include <unistd.h>
#include <cmath>


namespace amici {
namespace hdf5 {

/**
 * @brief assertMeasurementDimensionsCompatible
 * @param m
 * @param n
 * @param model
 */
void assertMeasurementDimensionsCompatible(hsize_t m, hsize_t n, Model const& model) {
    // if this is rank 1, n and m can be swapped
    if (n == 1) {
        assert(n == (unsigned)model.nt() || n == (unsigned)model.nytrue);
        assert(m == (unsigned)model.nytrue || m == (unsigned)model.nt());
        assert(m * n == (unsigned)model.nytrue * model.nt());
    } else {
        assert(n == (unsigned)model.nt());
        assert(m == (unsigned)model.nytrue);
    }
}


/**
 * @brief assertEventDimensionsCompatible
 * @param m
 * @param n
 * @param model
 */
void assertEventDimensionsCompatible(hsize_t m, hsize_t n, Model const& model) {
    // if this is rank 1, n and m can be swapped
    if (n == 1) {
        assert(n == (unsigned)model.nMaxEvent() ||
               n == (unsigned)model.nztrue);
        assert(m == (unsigned)model.nztrue ||
               m == (unsigned)model.nMaxEvent());
        assert(m * n == (unsigned)model.nytrue * model.nMaxEvent());
    } else {
        assert(n == (unsigned)model.nMaxEvent());
        assert(m == (unsigned)model.nztrue);
    }
}


void createGroup(H5::H5File &file,
                 std::string const& groupPath,
                 bool recursively) {

    hid_t groupCreationPropertyList = H5P_DEFAULT;

    if (recursively) {
        groupCreationPropertyList = H5Pcreate(H5P_LINK_CREATE);
        H5Pset_create_intermediate_group(groupCreationPropertyList, 1);
    }

    hid_t group = H5Gcreate(file.getId(), groupPath.c_str(),
                            groupCreationPropertyList,
                            H5P_DEFAULT, H5P_DEFAULT);
    H5Pclose(groupCreationPropertyList);

    if (group < 0)
        throw(AmiException("Failed to create group in hdf5CreateGroup: %s", groupPath.c_str()));
    H5Gclose(group);
}

std::unique_ptr<ExpData> readSimulationExpData(std::string const& hdf5Filename,
                                               std::string const& hdf5Root,
                                               Model const& model) {
    H5::H5File file(hdf5Filename, H5F_ACC_RDONLY);

    hsize_t m, n;

    auto edata = std::unique_ptr<ExpData>(new ExpData(model));

    auto my = getDoubleArrayAttribute2D(file, hdf5Root, "Y", m, n);
    if(m * n > 0) {
        assertMeasurementDimensionsCompatible(m, n, model);
        edata->my = my;
    }

    auto sigmay = getDoubleArrayAttribute2D(file, hdf5Root, "Sigma_Y", m, n);
    if(m * n > 0) {
        assertMeasurementDimensionsCompatible(m, n, model);
        edata->sigmay = sigmay;
    }

    if (model.nz) {
        edata->mz = getDoubleArrayAttribute2D(file, hdf5Root, "Z", m, n);
        assertEventDimensionsCompatible(m, n, model);

        edata->sigmaz = getDoubleArrayAttribute2D(file, hdf5Root, "Sigma_Z", m, n);
        assertEventDimensionsCompatible(m, n, model);
    }

    return edata;
}

void writeReturnData(ReturnData const& rdata,
                     std::string const& hdf5Filename,
                     std::string const& hdf5Location) {
    auto file = createOrOpenForWriting(hdf5Filename);

    if(!locationExists(file, hdf5Location))
        createGroup(file, hdf5Location);

    if (rdata.ts)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "t",
                                 rdata.ts, rdata.nt);

    if (rdata.xdot)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "xdot", rdata.xdot,
                                 rdata.nx);

    if (rdata.llh)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "llh", rdata.llh, 1);
    
    if (rdata.status)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "status", rdata.status, 1);

    if (rdata.sllh)
        H5LTset_attribute_double(file.getId(), hdf5Location.c_str(), "sllh", rdata.sllh,
                                 rdata.nplist);

    // are double, but should write as int:
    if (rdata.numsteps)
        setAttributeIntFromDouble(file, hdf5Location, "numsteps",
                                  rdata.numsteps, rdata.nt);

    if (rdata.numrhsevals)
        setAttributeIntFromDouble(file, hdf5Location, "numrhsevals",
                                  rdata.numrhsevals, rdata.nt);

    if (rdata.numerrtestfails)
        setAttributeIntFromDouble(file, hdf5Location,
                                  "numerrtestfails",
                                  rdata.numerrtestfails, rdata.nt);

    if (rdata.numnonlinsolvconvfails)
        setAttributeIntFromDouble(
                    file, hdf5Location, "numnonlinsolvconvfails",
                    rdata.numnonlinsolvconvfails, rdata.nt);

    if (rdata.order)
        setAttributeIntFromDouble(file, hdf5Location, "order",
                                  rdata.order, rdata.nt);

    if (rdata.numstepsB)
        setAttributeIntFromDouble(file, hdf5Location, "numstepsB",
                                  rdata.numstepsB, rdata.nt);

    if (rdata.numrhsevalsB)
        setAttributeIntFromDouble(file, hdf5Location, "numrhsevalsB",
                                  rdata.numrhsevalsB, rdata.nt);

    if (rdata.numerrtestfailsB)
        setAttributeIntFromDouble(file, hdf5Location,
                                  "numerrtestfailsB",
                                  rdata.numerrtestfailsB, rdata.nt);

    if (rdata.numnonlinsolvconvfailsB)
        setAttributeIntFromDouble(
                    file, hdf5Location, "numnonlinsolvconvfailsB",
                    rdata.numnonlinsolvconvfailsB, rdata.nt);

    auto group = file.openGroup(hdf5Location);
{'fraction_non_alphanumeric': 0.1188879010749212, 'fraction_numerical': 0.010506748565424716, 'mean_word_length': 2.3864258347016967, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803793', 'n_tokens_mistral': 4125, 'n_tokens_neox': 4037, 'n_words': 748}