<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Runtime.Serialization;
using System.Text;

namespace KelpNet
{
    public class LabeledDataSet
    {
        NetDataContractSerializer bf = new NetDataContractSerializer();
        private ZipArchive ZipArchiveData;
        private bool[] Loaded;

        private Real[][] Data;
        private Real[] DataLabel;

        private int[] trainIndex;
        private int[] validIndex;
        private int[] testIndex;

        public int Length = 0;
        public string[] LabelNames;
        public int[] Shape;

        public LabeledDataSet(Real[][] data, Real[] dataLabel, int[] shape, string[] labelNames = null, bool makeValidData = false, bool makeTrainIndex = true, int augmentCount = 1)
        {
            Data = data;
            DataLabel = dataLabel;
            Loaded = Enumerable.Repeat(true, data.Length).ToArray();
            LabelNames = labelNames;
            Shape = shape;
            Length = data.Length;

            //ラベル名称が未指定の場合、連番を自動的に割り当てる
            if (LabelNames == null)
            {
                Real maxLabel = 0;

                for (int i = 0; i < DataLabel.Length; i++)
                {
                    if (DataLabel[i] > maxLabel) maxLabel = DataLabel[i];
                }

                LabelNames = Enumerable.Range(0, (int)maxLabel).Select(s => s.ToString()).ToArray();
            }

            if (makeTrainIndex)
            {
                MakeTrainData(makeValidData, augmentCount);
            }
        }

        //Train,Valid,Testデータを作成する
        void MakeTrainData(bool makeValidData, int augmentCount = 1)
        {
            //全データのインデックスを最初に持っておく
            List<int> trainIndexList = new List<int>(Enumerable.Range(0, DataLabel.Length));

            //各クラスで水増し前の10％のデータをテストデータに割り当てる
            int testDataCount = DataLabel.Length / LabelNames.Length / 10 / augmentCount;

            //少なすぎる場合は一つ
            if (testDataCount == 0) testDataCount = 1;

            //テストデータ用のIndexを作成
            testIndex = GetRondomIndex(testDataCount, LabelNames.Length, augmentCount, trainIndexList);
<commit_msg>名称の自動割付に失敗するバグを修正<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Runtime.Serialization;
using System.Text;

namespace KelpNet
{
    public class LabeledDataSet
    {
        NetDataContractSerializer bf = new NetDataContractSerializer();
        private ZipArchive ZipArchiveData;
        private bool[] Loaded;

        private Real[][] Data;
        private Real[] DataLabel;

        private int[] trainIndex;
        private int[] validIndex;
        private int[] testIndex;

        public int Length = 0;
        public string[] LabelNames;
        public int[] Shape;

        public LabeledDataSet(Real[][] data, Real[] dataLabel, int[] shape, string[] labelNames = null, bool makeValidData = false, bool makeTrainIndex = true, int augmentCount = 1)
        {
            Data = data;
            DataLabel = dataLabel;
            Loaded = Enumerable.Repeat(true, data.Length).ToArray();
            LabelNames = labelNames;
            Shape = shape;
            Length = data.Length;

            //ラベル名称が未指定の場合、連番を自動的に割り当てる
            if (LabelNames == null)
            {
                Real maxLabel = 0;

                for (int i = 0; i < DataLabel.Length; i++)
                {
                    if (DataLabel[i] > maxLabel) maxLabel = DataLabel[i];
                }

                LabelNames = Enumerable.Range(0, (int)maxLabel + 1).Select(s => s.ToString()).ToArray();
            }

            if (makeTrainIndex)
            {
                MakeTrainData(makeValidData, augmentCount);
            }
        }

        //Train,Valid,Testデータを作成する
        void MakeTrainData(bool makeValidData, int augmentCount = 1)
        {
            //全データのインデックスを最初に持っておく
            List<int> trainIndexList = new List<int>(Enumerable.Range(0, DataLabel.Length));

            //各クラスで水増し前の10％のデータをテストデータに割り当てる
            int testDataCount = DataLabel.Length / LabelNames.Length / 10 / augmentCount;

            //少なすぎる場合は一つ
            if (testDataCount == 0) testDataCount = 1;

            //テストデータ用のIndexを作成
            testIndex = GetRondomIndex(testDataCount, LabelNames.Length, augmentCount, trainIndexList);

{'fraction_non_alphanumeric': 0.09447935368043088, 'fraction_numerical': 0.006059245960502693, 'mean_word_length': 1.9733155436957972, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '509873', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1325, 'n_words': 291}