<commit_before>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

#include <nix/hdf5/DataSetHDF5.hpp>
#include <nix/hdf5/ExceptionHDF5.hpp>

#include <iostream>
#include <cmath>

namespace nix {


//special to_data_type for our purporses (Value)
template<>
struct to_data_type<char *> {
    static const bool is_valid = true;
    static const DataType value = DataType::String;
};

template<>
struct to_data_type<const char *> {
    static const bool is_valid = true;
    static const DataType value = DataType::String;
};

namespace hdf5 {

DataSet::DataSet(hid_t hid)
        : LocID(hid) {

}

DataSet::DataSet(const DataSet &other)
        : LocID(other)  {

}

void DataSet::read(hid_t memType, void *data) const
{
    HErr res = H5Dread(hid, memType, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    res.check("DataSet::read() IO error");
}

void DataSet::write(hid_t memType, const void *data)
{
    HErr res = H5Dwrite(hid, memType, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    res.check("DataSet::write() IOError");
}

void DataSet::read(DataType dtype, const NDSize &size, void *data) const
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);

    if (dtype == DataType::String) {
        StringWriter writer(size, static_cast<std::string *>(data));
        read(memType.h5id(), *writer);
        writer.finish();
        vlenReclaim(memType.h5id(), *writer);
    } else {
        read(memType.h5id(), data);
    }
}


void DataSet::write(DataType dtype, const NDSize &size, const void *data)
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);
    if (dtype == DataType::String) {
        StringReader reader(size, static_cast<const std::string *>(data));
        write(memType.h5id(), *reader);
    } else {
        write(memType.h5id(), data);
    }
}


void DataSet::read(DataType        dtype,
                   void            *data,
                   const Selection &fileSel,
                   const Selection &memSel) const
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);

    HErr res;
    if (dtype == DataType::String) {
        NDSize size = memSel.size();
        StringWriter writer(size, static_cast<std::string *>(data));
        res = H5Dread(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, *writer);
        writer.finish();
        vlenReclaim(memType.h5id(), *writer);
    } else {
        res = H5Dread(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, data);
    }

    res.check("DataSet::read() IO error");
}

void DataSet::write(DataType         dtype,
                    const void      *data,
                    const Selection &fileSel,
                    const Selection &memSel)
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);
    HErr res;

    if (dtype == DataType::String) {
        NDSize size = memSel.size();
        StringReader reader(size, static_cast<const std::string *>(data));
        res = H5Dwrite(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, *reader);
    } else {
        res = H5Dwrite(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, data);
    }

    res.check("DataSet::write(): IO error");
}


double psize_product(const NDSize &dims)
{
    double product = 1;
    std::for_each(dims.begin(), dims.end(), [&](hsize_t val) {
        product *= val;
    });

    return product;
}


#define CHUNK_BASE   16*1024
#define CHUNK_MIN     8*1024
#define CHUNK_MAX  1024*1024

/**
 * Infer the chunk size from the supplied size information
 *
 * @param dims    Size information to base the guessing on
 * @param dtype   The type of the data to guess the chunks for
 *
 * Internally uses guessChunking(NDSize, size_t) for calculations.
 *
 * @return An (maybe not at all optimal) guess for chunk size
 */
NDSize DataSet::guessChunking(NDSize dims, DataType dtype)
{
    const size_t type_size = data_type_to_size(dtype);
    NDSize chunks = guessChunking(dims, type_size);
    return chunks;
}

/**<commit_msg>DataSet: remove unused psize_product function<commit_after>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

#include <nix/hdf5/DataSetHDF5.hpp>
#include <nix/hdf5/ExceptionHDF5.hpp>

#include <iostream>
#include <cmath>

namespace nix {


//special to_data_type for our purporses (Value)
template<>
struct to_data_type<char *> {
    static const bool is_valid = true;
    static const DataType value = DataType::String;
};

template<>
struct to_data_type<const char *> {
    static const bool is_valid = true;
    static const DataType value = DataType::String;
};

namespace hdf5 {

DataSet::DataSet(hid_t hid)
        : LocID(hid) {

}

DataSet::DataSet(const DataSet &other)
        : LocID(other)  {

}

void DataSet::read(hid_t memType, void *data) const
{
    HErr res = H5Dread(hid, memType, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    res.check("DataSet::read() IO error");
}

void DataSet::write(hid_t memType, const void *data)
{
    HErr res = H5Dwrite(hid, memType, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    res.check("DataSet::write() IOError");
}

void DataSet::read(DataType dtype, const NDSize &size, void *data) const
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);

    if (dtype == DataType::String) {
        StringWriter writer(size, static_cast<std::string *>(data));
        read(memType.h5id(), *writer);
        writer.finish();
        vlenReclaim(memType.h5id(), *writer);
    } else {
        read(memType.h5id(), data);
    }
}


void DataSet::write(DataType dtype, const NDSize &size, const void *data)
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);
    if (dtype == DataType::String) {
        StringReader reader(size, static_cast<const std::string *>(data));
        write(memType.h5id(), *reader);
    } else {
        write(memType.h5id(), data);
    }
}


void DataSet::read(DataType        dtype,
                   void            *data,
                   const Selection &fileSel,
                   const Selection &memSel) const
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);

    HErr res;
    if (dtype == DataType::String) {
        NDSize size = memSel.size();
        StringWriter writer(size, static_cast<std::string *>(data));
        res = H5Dread(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, *writer);
        writer.finish();
        vlenReclaim(memType.h5id(), *writer);
    } else {
        res = H5Dread(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, data);
    }

    res.check("DataSet::read() IO error");
}

void DataSet::write(DataType         dtype,
                    const void      *data,
                    const Selection &fileSel,
                    const Selection &memSel)
{
    h5x::DataType memType = data_type_to_h5_memtype(dtype);
    HErr res;

    if (dtype == DataType::String) {
        NDSize size = memSel.size();
        StringReader reader(size, static_cast<const std::string *>(data));
        res = H5Dwrite(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, *reader);
    } else {
        res = H5Dwrite(hid, memType.h5id(), memSel.h5space().h5id(), fileSel.h5space().h5id(), H5P_DEFAULT, data);
    }

    res.check("DataSet::write(): IO error");
}

#define CHUNK_BASE   16*1024
#define CHUNK_MIN     8*1024
#define CHUNK_MAX  1024*1024

/**
 * Infer the chunk size from the supplied size information
 *
 * @param dims    Size information to base the guessing on
 * @param dtype   The type of the data to guess the chunks for
 *
 * Internally uses guessChunking(NDSize, size_t) for calculations.
 *
 * @return An (maybe not at all optimal) guess for chunk size
 */
NDSize DataSet::guessChunking(NDSize dims, DataType dtype)
{
    const size_t type_size = data_type_to_size(dtype);
    NDSize chunks = guessChunking(dims, type_size);
    return chunks;
}

/**
{'fraction_non_alphanumeric': 0.13174791914387635, 'fraction_numerical': 0.018668252080856124, 'mean_word_length': 3.213927855711423, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2817081', 'n_tokens_mistral': 2916, 'n_tokens_neox': 2883, 'n_words': 705}