<commit_before>//
// Created by kanairen on 2016/09/15.
//

#ifndef CONVNETCPP_DATASET_H
#define CONVNETCPP_DATASET_H

#include <sstream>
#include <vector>
#include <algorithm>
#include <random>

using std::vector;
using std::unique_ptr;

template<class X, class Y>
class BaseDataSet {

    /*
     * データセット基底クラス
     */

public:

    unique_ptr<vector<vector<X>>> x_train;
    unique_ptr<vector<vector<X>>> x_test;
    unique_ptr<vector<Y>> y_train;
    unique_ptr<vector<Y>> y_test;

protected:

    BaseDataSet(vector<vector<X>> *x_train, vector<vector<X>> *x_test,
                vector<Y> *y_train, vector<Y> *y_test)
            : x_train() { }

    static void shuffle(const unique_ptr<vector<vector<X>>> &x,
                        const unique_ptr<vector<Y>> &y) {

        /*
         * 特徴ベクトル集合と正解データ集合を、特徴ベクトル・正解の関係が崩れないようシャッフルする
         */

        vector<int> indices(y->size());

        for (int i = 0; i < indices.size(); ++i) {
            indices[i] = i;
        }

        std::shuffle(indices.begin(), indices.end(), std::mt19937());

        vector<vector<X>> tmp_x(indices.size());
        vector<Y> tmp_y(indices.size());

        for (int i = 0; i < indices.size(); ++i) {
            tmp_x[i] = (*x)[indices[i]];
            tmp_y[i] = (*y)[indices[i]];
        }

        for (int i = 0; i < indices.size(); ++i) {
            (*x)[i] = tmp_x[i];
            (*y)[i] = tmp_y[i];
        }

    }

public:
    void shuffle(bool is_train) {
        if (is_train) {
            BaseDataSet::shuffle(x_train, y_train);
        } else {
            BaseDataSet::shuffle(x_test, y_test);
        }
    }

    virtual unsigned int data_size() = 0;

    static const unique_ptr<BaseDataSet> load(){

    }

    static const unique_ptr<vector<BaseDataSet>> cross_validation(
            unsigned int n_fold,
            const vector<vector<X>> &x,
            const vector<Y> &y) {
        /*
         * n-fold交差検定のデータ・セットを返す
         */

        vector<BaseDataSet> *data_sets = new vector<DataSet>();

        // shuffle<commit_msg>fix BaseDataSet::shuffle()<commit_after>//
// Created by kanairen on 2016/09/15.
//

#ifndef CONVNETCPP_DATASET_H
#define CONVNETCPP_DATASET_H

#include <sstream>
#include <vector>
#include <algorithm>
#include <random>

using std::vector;
using std::unique_ptr;

template<class X, class Y>
class BaseDataSet {

    /*
     * データセット基底クラス
     */

public:

    unique_ptr<vector<vector<X>>> x_train;
    unique_ptr<vector<vector<X>>> x_test;
    unique_ptr<vector<Y>> y_train;
    unique_ptr<vector<Y>> y_test;

protected:

    BaseDataSet(vector<vector<X>> *x_train, vector<vector<X>> *x_test,
                vector<Y> *y_train, vector<Y> *y_test)
            : x_train() { }

    static void shuffle(const unique_ptr<vector<vector<X>>> &x,
                        const unique_ptr<vector<Y>> &y) {

        /*
         * 特徴ベクトル集合と正解データ集合を、特徴ベクトル・正解の関係が崩れないようシャッフルする
         */

        vector<int> indices(y->size());

        for (int i = 0; i < indices.size(); ++i) {
            indices[i] = i;
        }

        std::shuffle(indices.begin(), indices.end(), std::mt19937());

        vector<vector<X>> tmp_x(x->size(), vector<X>((*x[0]).size()));
        vector<Y> tmp_y(y->size());

        for (int i = 0; i < indices.size(); ++i) {
            for (int j = 0; j < x->size(); ++j) {
                tmp_x[j][i] = (*x)[j][indices[i]];
            }
            tmp_y[i] = (*y)[indices[i]];
        }

        for (int i = 0; i < indices.size(); ++i) {
            for (int j = 0; j < x.size(); ++j) {
                (*x)[j][i] = tmp_x[j][i];
            }
            (*y)[i] = tmp_y[i];
        }

    }

public:
    void shuffle(bool is_train) {
        if (is_train) {
            BaseDataSet::shuffle(x_train, y_train);
        } else {
            BaseDataSet::shuffle(x_test, y_test);
        }
    }

    virtual unsigned int data_size() = 0;

    static const unique_ptr<BaseDataSet> load(){

    }

    static const unique_ptr<vector<BaseDataSet>> cross_validation(
            unsigned int n_fold,
            const vector<vector<X>> &x,
            const vector<Y> &y) {
        /*
         * n-fold交差検定のデータ・セットを返す
         */

        vector<BaseDataSet> *data_sets = new vector<DataSet>();

        // shuffle
{'fraction_non_alphanumeric': 0.15822637106184365, 'fraction_numerical': 0.008634772462077013, 'mean_word_length': 2.1376281112737923, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965868', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1567, 'n_words': 287}