<commit_before>// @(#)root/tmva $Id$
// Author: <NAME>

/*************************************************************************
 * Copyright (C) 2018, <NAME> and <NAME>.                    *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#include "TMVA/CvSplit.h"

#include "TMVA/DataSet.h"
#include "TMVA/DataSetFactory.h"
#include "TMVA/DataSetInfo.h"
#include "TMVA/Event.h"
#include "TMVA/MsgLogger.h"
#include "TMVA/Tools.h"

#include <TString.h>
#include <TFormula.h>

#include <algorithm>
#include <numeric>
#include <stdexcept>

ClassImp(TMVA::CvSplit);
ClassImp(TMVA::CvSplitKFolds);

/* =============================================================================
      TMVA::CvSplit
============================================================================= */

////////////////////////////////////////////////////////////////////////////////
///

TMVA::CvSplit::CvSplit(UInt_t numFolds) : fNumFolds(numFolds), fMakeFoldDataSet(kFALSE) {}

////////////////////////////////////////////////////////////////////////////////
/// \brief Set training and test set vectors of dataset described by `dsi`.
/// \param[in] dsi DataSetInfo for data set to be split
/// \param[in] foldNumber Ordinal of fold to prepare
/// \param[in] tt The set used to prepare fold. If equal to `Types::kTraining`
///               splitting will be based off the original train set. If instead
///               equal to `Types::kTesting` the test set will be used.
///               The original training/test set is the set as defined by
///               `DataLoader::PrepareTrainingAndTestSet`.
///
/// Sets the training and test set vectors of the DataSet described by `dsi` as
/// defined by the split. If `tt` is eqal to `Types::kTraining` the split will
/// be based off of the original training set.
///
/// Note: Requires `MakeKFoldDataSet` to have been called first.
///

void TMVA::CvSplit::PrepareFoldDataSet(DataSetInfo &dsi, UInt_t foldNumber, Types::ETreeType tt)
{
   if (foldNumber >= fNumFolds) {
      Log() << kFATAL << "DataSet prepared for \"" << fNumFolds << "\" folds, requested fold \"" << foldNumber
            << "\" is outside of range." << Endl;
      return;
   }

   auto prepareDataSetInternal = [this, &dsi, foldNumber](std::vector<std::vector<Event *>> vec) {
      UInt_t numFolds = fTrainEvents.size();

      // Events in training set (excludes current fold)
      UInt_t nTotal = std::accumulate(vec.begin(), vec.end(), 0,
                                      [&](UInt_t sum, std::vector<TMVA::Event *> v) { return sum + v.size(); });

      UInt_t nTrain = nTotal - vec.at(foldNumber).size();
      UInt_t nTest = vec.at(foldNumber).size();

      std::vector<Event *> tempTrain;
      std::vector<Event *> tempTest;

      tempTrain.reserve(nTrain);
      tempTest.reserve(nTest);

      // Insert data into training set
      for (UInt_t i = 0; i < numFolds; ++i) {
         if (i == foldNumber) {
            continue;
         }

         tempTrain.insert(tempTrain.end(), vec.at(i).begin(), vec.at(i).end());
      }

      // Insert data into test set
      tempTest.insert(tempTest.end(), vec.at(foldNumber).begin(), vec.at(foldNumber).end());

      Log() << kDEBUG << "Fold prepared, num events in training set: " << tempTrain.size() << Endl;
      Log() << kDEBUG << "Fold prepared, num events in test     set: " << tempTest.size() << Endl;

      // Assign the vectors of the events to rebuild the dataset
      dsi.GetDataSet()->SetEventCollection(&tempTrain, Types::kTraining, false);
      dsi.GetDataSet()->SetEventCollection(&tempTest, Types::kTesting, false);
   };

   if (tt == Types::kTraining) {
      prepareDataSetInternal(fTrainEvents);
   } else if (tt == Types::kTesting) {
      prepareDataSetInternal(fTestEvents);
   } else {
      Log() << kFATAL << "PrepareFoldDataSet can only work with training and testing data sets." << std::endl;
      return;
   }
}

////////////////////////////////////////////////////////////////////////////////
///

void TMVA::CvSplit::RecombineKFoldDataSet(DataSetInfo &dsi, Types::ETreeType tt)
{
   if (tt != Types::kTraining) {
      Log() << kFATAL << "Only kTraining is supported for CvSplit::RecombineKFoldDataSet currently." << std::endl;
   }

   std::vector<Event *> *tempVec = new std::vector<Event *>;

   for (UInt_t i = 0; i < fNumFolds; ++i) {
      tempVec->insert(tempVec->end(), fTrainEvents.at(i).begin(), fTrainEvents.at(i).end());
   }

   dsi.GetDataSet()->SetEventCollection(tempVec, Types::kTraining, false);
   dsi.GetDataSet()->SetEventCollection(tempVec, Types::kTesting, false);

   delete tempVec;
}

/* =============================================================================
      TMVA::CvSplitKFoldsExpr
============================================================================= */

////////////////////////////////////////////////////////////////////////////////
///

TMVA::CvSplitKFoldsExpr::CvSplitKFoldsExpr(DataSetInfo &dsi, TString expr)
   : fDsi(dsi), fIdxFormulaParNumFolds(std::numeric_limits<UInt_t>::max()), fSplitFormula("", expr),
     fParValues(fSplitFormula.GetNpar())
{
   if (not fSplitFormula.IsValid()) {
      throw std::runtime_error("Split expression \"" + std::string(fSplitExpr.Data()) + "\" is not a valid TFormula.");
   }

   for (Int_t iFormulaPar = 0; iFormulaPar < fSplitFormula.GetNpar(); ++iFormulaPar) {
      TString name = fSplitFormula.GetParName(iFormulaPar);

      // std::cout << "Found variable with name \"" << name << "\"." << std::endl;

      if (name == "NumFolds" or name == "numFolds") {
         // std::cout << "NumFolds|numFolds is a reserved variable! Adding to context." << std::endl;
         fIdxFormulaParNumFolds = iFormulaPar;
      } else {
         fFormulaParIdxToDsiSpecIdx.push_back(std::make_pair(iFormulaPar, GetSpectatorIndexForName(fDsi, name)));
      }
   }
}

////////////////////////////////////////////////////////////////////////////////
///

UInt_t TMVA::CvSplitKFoldsExpr::Eval(UInt_t numFolds, const Event *ev)
{
   for (auto &p : fFormulaParIdxToDsiSpecIdx) {
      auto iFormulaPar = p.first;
      auto iSpectator = p.second;

      fParValues.at(iFormulaPar) = ev->GetSpectator(iSpectator);
   }

   if (fIdxFormulaParNumFolds < fSplitFormula.GetNpar()) {
      fParValues[fIdxFormulaParNumFolds] = numFolds;
   }

   Double_t iFold = fSplitFormula.EvalPar(nullptr, &fParValues[0]);

   if (fabs(iFold - (double)((UInt_t)iFold)) > 1e-5) {
      throw std::runtime_error(
         "Output of splitExpr should be a non-negative integer between 0 and numFolds-1 inclusive.");
   }
<commit_msg>[TMVA] CVSplit -- Improve fold assignment for large values<commit_after>// @(#)root/tmva $Id$
// Author: <NAME>

/*************************************************************************
 * Copyright (C) 2018, <NAME> and <NAME>.                    *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#include "TMVA/CvSplit.h"

#include "TMVA/DataSet.h"
#include "TMVA/DataSetFactory.h"
#include "TMVA/DataSetInfo.h"
#include "TMVA/Event.h"
#include "TMVA/MsgLogger.h"
#include "TMVA/Tools.h"

#include <TString.h>
#include <TFormula.h>

#include <algorithm>
#include <numeric>
#include <stdexcept>

ClassImp(TMVA::CvSplit);
ClassImp(TMVA::CvSplitKFolds);

/* =============================================================================
      TMVA::CvSplit
============================================================================= */

////////////////////////////////////////////////////////////////////////////////
///

TMVA::CvSplit::CvSplit(UInt_t numFolds) : fNumFolds(numFolds), fMakeFoldDataSet(kFALSE) {}

////////////////////////////////////////////////////////////////////////////////
/// \brief Set training and test set vectors of dataset described by `dsi`.
/// \param[in] dsi DataSetInfo for data set to be split
/// \param[in] foldNumber Ordinal of fold to prepare
/// \param[in] tt The set used to prepare fold. If equal to `Types::kTraining`
///               splitting will be based off the original train set. If instead
///               equal to `Types::kTesting` the test set will be used.
///               The original training/test set is the set as defined by
///               `DataLoader::PrepareTrainingAndTestSet`.
///
/// Sets the training and test set vectors of the DataSet described by `dsi` as
/// defined by the split. If `tt` is eqal to `Types::kTraining` the split will
/// be based off of the original training set.
///
/// Note: Requires `MakeKFoldDataSet` to have been called first.
///

void TMVA::CvSplit::PrepareFoldDataSet(DataSetInfo &dsi, UInt_t foldNumber, Types::ETreeType tt)
{
   if (foldNumber >= fNumFolds) {
      Log() << kFATAL << "DataSet prepared for \"" << fNumFolds << "\" folds, requested fold \"" << foldNumber
            << "\" is outside of range." << Endl;
      return;
   }

   auto prepareDataSetInternal = [this, &dsi, foldNumber](std::vector<std::vector<Event *>> vec) {
      UInt_t numFolds = fTrainEvents.size();

      // Events in training set (excludes current fold)
      UInt_t nTotal = std::accumulate(vec.begin(), vec.end(), 0,
                                      [&](UInt_t sum, std::vector<TMVA::Event *> v) { return sum + v.size(); });

      UInt_t nTrain = nTotal - vec.at(foldNumber).size();
      UInt_t nTest = vec.at(foldNumber).size();

      std::vector<Event *> tempTrain;
      std::vector<Event *> tempTest;

      tempTrain.reserve(nTrain);
      tempTest.reserve(nTest);

      // Insert data into training set
      for (UInt_t i = 0; i < numFolds; ++i) {
         if (i == foldNumber) {
            continue;
         }

         tempTrain.insert(tempTrain.end(), vec.at(i).begin(), vec.at(i).end());
      }

      // Insert data into test set
      tempTest.insert(tempTest.end(), vec.at(foldNumber).begin(), vec.at(foldNumber).end());

      Log() << kDEBUG << "Fold prepared, num events in training set: " << tempTrain.size() << Endl;
      Log() << kDEBUG << "Fold prepared, num events in test     set: " << tempTest.size() << Endl;

      // Assign the vectors of the events to rebuild the dataset
      dsi.GetDataSet()->SetEventCollection(&tempTrain, Types::kTraining, false);
      dsi.GetDataSet()->SetEventCollection(&tempTest, Types::kTesting, false);
   };

   if (tt == Types::kTraining) {
      prepareDataSetInternal(fTrainEvents);
   } else if (tt == Types::kTesting) {
      prepareDataSetInternal(fTestEvents);
   } else {
      Log() << kFATAL << "PrepareFoldDataSet can only work with training and testing data sets." << std::endl;
      return;
   }
}

////////////////////////////////////////////////////////////////////////////////
///

void TMVA::CvSplit::RecombineKFoldDataSet(DataSetInfo &dsi, Types::ETreeType tt)
{
   if (tt != Types::kTraining) {
      Log() << kFATAL << "Only kTraining is supported for CvSplit::RecombineKFoldDataSet currently." << std::endl;
   }

   std::vector<Event *> *tempVec = new std::vector<Event *>;

   for (UInt_t i = 0; i < fNumFolds; ++i) {
      tempVec->insert(tempVec->end(), fTrainEvents.at(i).begin(), fTrainEvents.at(i).end());
   }

   dsi.GetDataSet()->SetEventCollection(tempVec, Types::kTraining, false);
   dsi.GetDataSet()->SetEventCollection(tempVec, Types::kTesting, false);

   delete tempVec;
}

/* =============================================================================
      TMVA::CvSplitKFoldsExpr
============================================================================= */

////////////////////////////////////////////////////////////////////////////////
///

TMVA::CvSplitKFoldsExpr::CvSplitKFoldsExpr(DataSetInfo &dsi, TString expr)
   : fDsi(dsi), fIdxFormulaParNumFolds(std::numeric_limits<UInt_t>::max()), fSplitFormula("", expr),
     fParValues(fSplitFormula.GetNpar())
{
   if (not fSplitFormula.IsValid()) {
      throw std::runtime_error("Split expression \"" + std::string(fSplitExpr.Data()) + "\" is not a valid TFormula.");
   }

   for (Int_t iFormulaPar = 0; iFormulaPar < fSplitFormula.GetNpar(); ++iFormulaPar) {
      TString name = fSplitFormula.GetParName(iFormulaPar);

      // std::cout << "Found variable with name \"" << name << "\"." << std::endl;

      if (name == "NumFolds" or name == "numFolds") {
         // std::cout << "NumFolds|numFolds is a reserved variable! Adding to context." << std::endl;
         fIdxFormulaParNumFolds = iFormulaPar;
      } else {
         fFormulaParIdxToDsiSpecIdx.push_back(std::make_pair(iFormulaPar, GetSpectatorIndexForName(fDsi, name)));
      }
   }
}

////////////////////////////////////////////////////////////////////////////////
///

UInt_t TMVA::CvSplitKFoldsExpr::Eval(UInt_t numFolds, const Event *ev)
{
   for (auto &p : fFormulaParIdxToDsiSpecIdx) {
      auto iFormulaPar = p.first;
      auto iSpectator = p.second;

      fParValues.at(iFormulaPar) = ev->GetSpectator(iSpectator);
   }

   if (fIdxFormulaParNumFolds < fSplitFormula.GetNpar()) {
      fParValues[fIdxFormulaParNumFolds] = numFolds;
   }

   // NOTE: We are using a double to represent an integer here. This _will_
   // lead to problems if the norm of the double grows too large. A quick test
   // with python suggests that problems arise at a magnitude of ~1e16.
   Double_t iFold_d = fSplitFormula.EvalPar(nullptr, &fParValues[0]);

   if (iFold_d < 0) {
      throw std::runtime_error("Output of splitExpr must be non-negative.");
   }

   UInt_t iFold = std::lround(iFold_d);
   if (iFold >= numFolds) {
      throw std::runtime_error("Output of splitExpr should be a non-negative"
                               "integer between 0 and numFolds-1 inclusive.");
   }

{'fraction_non_alphanumeric': 0.24609566496253238, 'fraction_numerical': 0.0019609216331675887, 'mean_word_length': 2.5198422479664777, 'pattern_counts': {'":': 0, '<': 152, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741993', 'n_tokens_mistral': 4277, 'n_tokens_neox': 4130, 'n_words': 1007}