<commit_before>#include <tiny_dnn/tiny_dnn.h>

#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

#include <numeric> // std::iota

namespace tinyseg {

typedef unsigned short label_image_t;
const int label_image_type = CV_16UC1;

struct training_dataset {
    std::vector<tiny_dnn::vec_t> inputs;
    std::vector<tiny_dnn::label_t> labels;
    //std::vector<tiny_dnn::vec_t> weights;

    void shuffle() {
        if (!is_valid()) {
            throw std::runtime_error("Training dataset is not valid");
        }
        std::vector<size_t> new_order(inputs.size());
        std::iota(new_order.begin(), new_order.end(), 0);
        std::shuffle(new_order.begin(), new_order.end(), std::mt19937{ std::random_device{}() });

        std::vector<tiny_dnn::vec_t> new_inputs(inputs.size());
        std::vector<tiny_dnn::label_t> new_labels(labels.size());

        for (size_t i = 0, end = inputs.size(); i < end; ++i) {
            new_inputs[i] = inputs[new_order[i]];
            new_labels[i] = labels[new_order[i]];
        }

        std::swap(inputs, new_inputs);
        std::swap(labels, new_labels);
    }

    std::pair<training_dataset, training_dataset> split(double first_percentage = 50.0) {
        std::pair<training_dataset, training_dataset> result;
        training_dataset shuffled = *this;
        shuffled.shuffle();
        size_t first_count = static_cast<size_t>(inputs.size() * first_percentage / 100.0);
        std::move(shuffled.inputs.begin(), shuffled.inputs.begin() + first_count, std::back_inserter(result.first.inputs));
        std::move(shuffled.labels.begin(), shuffled.labels.begin() + first_count, std::back_inserter(result.first.labels));
        std::move(shuffled.inputs.begin() + first_count, shuffled.inputs.end(), std::back_inserter(result.second.inputs));
        std::move(shuffled.labels.begin() + first_count, shuffled.labels.end(), std::back_inserter(result.second.labels));
        return result;
    }

    bool is_valid() {
        return inputs.size() == labels.size();
    }
};

struct sample {
    cv::Mat original_image;
    cv::Mat labels;
    std::vector<cv::Point> mask; // where are the labels valid?
};

sample load_image(const std::string& original_image_filename, const std::string& labels_filename, const std::vector<cv::Scalar>& label_colors) {
    sample sample;

    sample.original_image = cv::imread(original_image_filename, cv::IMREAD_GRAYSCALE);

    if (sample.original_image.data == nullptr) {
        throw std::runtime_error("Unable to read original image file");
    }

    const cv::Mat labels_mask = cv::imread(labels_filename);

    if (labels_mask.data == nullptr) {
        throw std::runtime_error("Unable to read labels image file");
    }

    cv::Size image_size = sample.original_image.size();

    if (labels_mask.size() != image_size) {
        throw std::runtime_error("Original and labels image size mismatch");
    }

    sample.labels.create(image_size, label_image_type);
    sample.labels.setTo(std::numeric_limits<label_image_t>::max());

    std::vector<cv::Point> mask;

    cv::Mat label_mask;<commit_msg>Reserve memory beforehand<commit_after>#include <tiny_dnn/tiny_dnn.h>

#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

#include <numeric> // std::iota

namespace tinyseg {

typedef unsigned short label_image_t;
const int label_image_type = CV_16UC1;

struct training_dataset {
    std::vector<tiny_dnn::vec_t> inputs;
    std::vector<tiny_dnn::label_t> labels;
    //std::vector<tiny_dnn::vec_t> weights;

    void shuffle() {
        if (!is_valid()) {
            throw std::runtime_error("Training dataset is not valid");
        }
        std::vector<size_t> new_order(inputs.size());
        std::iota(new_order.begin(), new_order.end(), 0);
        std::shuffle(new_order.begin(), new_order.end(), std::mt19937{ std::random_device{}() });

        std::vector<tiny_dnn::vec_t> new_inputs(inputs.size());
        std::vector<tiny_dnn::label_t> new_labels(labels.size());

        for (size_t i = 0, end = inputs.size(); i < end; ++i) {
            new_inputs[i] = inputs[new_order[i]];
            new_labels[i] = labels[new_order[i]];
        }

        std::swap(inputs, new_inputs);
        std::swap(labels, new_labels);
    }

    std::pair<training_dataset, training_dataset> split(double first_percentage = 50.0) {
        std::pair<training_dataset, training_dataset> result;
        training_dataset shuffled = *this;
        shuffled.shuffle();
        size_t first_count = static_cast<size_t>(inputs.size() * first_percentage / 100.0);
        std::move(shuffled.inputs.begin(), shuffled.inputs.begin() + first_count, std::back_inserter(result.first.inputs));
        std::move(shuffled.labels.begin(), shuffled.labels.begin() + first_count, std::back_inserter(result.first.labels));
        std::move(shuffled.inputs.begin() + first_count, shuffled.inputs.end(), std::back_inserter(result.second.inputs));
        std::move(shuffled.labels.begin() + first_count, shuffled.labels.end(), std::back_inserter(result.second.labels));
        return result;
    }

    bool is_valid() {
        return inputs.size() == labels.size();
    }
};

struct sample {
    cv::Mat original_image;
    cv::Mat labels;
    std::vector<cv::Point> mask; // where are the labels valid?
};

sample load_image(const std::string& original_image_filename, const std::string& labels_filename, const std::vector<cv::Scalar>& label_colors) {
    sample sample;

    sample.original_image = cv::imread(original_image_filename, cv::IMREAD_GRAYSCALE);

    if (sample.original_image.data == nullptr) {
        throw std::runtime_error("Unable to read original image file");
    }

    const cv::Mat labels_mask = cv::imread(labels_filename);

    if (labels_mask.data == nullptr) {
        throw std::runtime_error("Unable to read labels image file");
    }

    cv::Size image_size = sample.original_image.size();

    if (labels_mask.size() != image_size) {
        throw std::runtime_error("Original and labels image size mismatch");
    }

    sample.labels.create(image_size, label_image_type);
    sample.labels.setTo(std::numeric_limits<label_image_t>::max());

    sample.mask.reserve(image_size.area());

    std::vector<cv::Point> mask;
    mask.reserve(image_size.area());

    cv::Mat label_mask;
{'fraction_non_alphanumeric': 0.14860632633886628, 'fraction_numerical': 0.006263701847792045, 'mean_word_length': 3.8792971734148205, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3651798', 'n_tokens_mistral': 2114, 'n_tokens_neox': 2040, 'n_words': 365}