<commit_before>// @(#)root/tmva $Id$
// Author: <NAME>
// Mentors: <NAME>, <NAME>
//NOTE: Based on TMVA::Factory

/**********************************************************************************
 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis       *
 * Package: TMVA                                                                  *
 * Class  : DataLoader                                                            *
 * Web    : http://tmva.sourceforge.net                                           *
 *                                                                                *
 * Description:                                                                   *
 *      This is a class to load datasets into every booked method                 *
 *                                                                                *
 * Authors (alphabetical):                                                        *
 *      <NAME> <<EMAIL>> - CERN, Switzerland               *
 *      <NAME> <<EMAIL>>  - ITM/UdeA, Colombia                   *
 *      <NAME><<EMAIL>> - CERN, Switzerland                *
 *                                                                                *
 * Copyright (c) 2005-2015:                                                       *
 *      CERN, Switzerland                                                         *
 *      ITM/UdeA, Colombia                                                        *
 *                                                                                *
 * Redistribution and use in source and binary forms, with or without             *
 * modification, are permitted according to the terms listed in LICENSE           *
 * (http://tmva.sourceforge.net/LICENSE)                                          *
 **********************************************************************************/


/*! \class TMVA::DataLoader
\ingroup TMVA

*/

#include "TTree.h"
#include "TH2.h"
#include "TMatrixD.h"

#include "TMVA/DataLoader.h"
#include "TMVA/Config.h"
#include "TMVA/CvSplit.h"
#include "TMVA/Tools.h"
#include "TMVA/IMethod.h"
#include "TMVA/MethodBase.h"
#include "TMVA/DataInputHandler.h"
#include "TMVA/DataSetManager.h"
#include "TMVA/DataSetInfo.h"
#include "TMVA/MethodBoost.h"
#include "TMVA/MethodCategory.h"

#include "TMVA/VariableInfo.h"
#include "TMVA/VariableIdentityTransform.h"
#include "TMVA/VariableDecorrTransform.h"
#include "TMVA/VariablePCATransform.h"
#include "TMVA/VariableGaussTransform.h"
#include "TMVA/VariableNormalizeTransform.h"
#include "TMVA/VarTransformHandler.h"

ClassImp(TMVA::DataLoader);

////////////////////////////////////////////////////////////////////////////////
/*** Create a data loader
 \param[in] thedlName  name of DataLoader object. This name will be used as the
                       top directory name where the training results
                        (weights, i.e .XML and .C files) will be stored.
                       The results will be stored by default in the `theDlName/weights`
                       directory and relative to the current directory. If the directory is not existing,
                       a new one will be created automatically.
                       For using a different location (i.e. a different path to the current directory) one
                       can set an absolute path location in `TMVA::gConfig()::GetIONames().fWeightFileDirPrefix`
                       For example, by setting
~~~~~~~~~~~~~~~{.cpp}
   TMVA::gConfig()::GetIONames().fWeightFileDirPrefix = "/tmp";
   TMVA::gConfig()::GetIONames().fWeightFileDir = "myTrainigResults";
~~~~~~~~~~~~~~~
                       The training results will be stored in the `/tmp/thedlName/myTrainingResults`
                       directory.
**/

TMVA::DataLoader::DataLoader( TString thedlName)
: Configurable( ),
   fDataSetManager       ( NULL ), //DSMTEST
   fDataInputHandler     ( new DataInputHandler ),
   fTransformations      ( "I" ),
   fVerbose              ( kFALSE ),
   fDataAssignType       ( kAssignEvents ),
   fATreeEvent           (0)
{
   fDataSetManager = new DataSetManager( *fDataInputHandler ); // DSMTEST
   SetName(thedlName.Data());
   fLogger->SetSource("DataLoader");
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataLoader::~DataLoader( void )
{
   // destructor

   std::vector<TMVA::VariableTransformBase*>::iterator trfIt = fDefaultTrfs.begin();
   for (;trfIt != fDefaultTrfs.end(); ++trfIt) delete (*trfIt);

   delete fDataInputHandler;

   // destroy singletons
   //   DataSetManager::DestroyInstance(); // DSMTEST replaced by following line
   delete fDataSetManager; // DSMTEST

   // problem with call of REGISTER_METHOD macro ...
   //   ClassifierDataLoader::DestroyInstance();
   //   Types::DestroyInstance();
   //Tools::DestroyInstance();
   //Config::DestroyInstance();
}


////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::AddDataSet( DataSetInfo &dsi )
{
   return fDataSetManager->AddDataSetInfo(dsi); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::AddDataSet( const TString& dsiName )
{
   DataSetInfo* dsi = fDataSetManager->GetDataSetInfo(dsiName); // DSMTEST

   if (dsi!=0) return *dsi;

   return fDataSetManager->AddDataSetInfo(*(new DataSetInfo(dsiName))); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::GetDataSetInfo()
{
   return DefaultDataSetInfo(); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////
/// Transforms the variables and return a new DataLoader with the transformed
/// variables

TMVA::DataLoader* TMVA::DataLoader::VarTransform(TString trafoDefinition)
{
   TString trOptions = "0";
   TString trName = "None";
   if (trafoDefinition.Contains("(")) {

      // contains transformation parameters
      Ssiz_t parStart = trafoDefinition.Index( "(" );
      Ssiz_t parLen   = trafoDefinition.Index( ")", parStart )-parStart+1;

      trName = trafoDefinition(0,parStart);
      trOptions = trafoDefinition(parStart,parLen);
      trOptions.Remove(parLen-1,1);
      trOptions.Remove(0,1);
   }
   else
      trName = trafoDefinition;

   VarTransformHandler* handler = new VarTransformHandler(this);
   // variance threshold variable transformation
   if (trName == "VT") {

      // find threshold value from given input
      Double_t threshold = 0.0;
      if (!trOptions.IsFloat()){
         Log() << kFATAL << " VT transformation must be passed a floating threshold value" << Endl;
         delete handler;
         return this;
      }
      else
         threshold =  trOptions.Atof();
      TMVA::DataLoader *transformedLoader = handler->VarianceThreshold(threshold);
      delete handler;
      return transformedLoader;
   }
   else {
      Log() << kFATAL << "Incorrect transformation string provided, please check" << Endl;
   }
   Log() << kINFO << "No transformation applied, returning original loader" << Endl;
   return this;
}

////////////////////////////////////////////////////////////////////////////////
// the next functions are to assign events directly

////////////////////////////////////////////////////////////////////////////////
/// create the data assignment tree (for event-wise data assignment by user)

TTree* TMVA::DataLoader::CreateEventAssignTrees( const TString& name )
{
   TTree * assignTree = new TTree( name, name );
   assignTree->SetDirectory(0);
   assignTree->Branch( "type",   &fATreeType,   "ATreeType/I" );
   assignTree->Branch( "weight", &fATreeWeight, "ATreeWeight/F" );

   std::vector<VariableInfo>& vars = DefaultDataSetInfo().GetVariableInfos();
   std::vector<VariableInfo>& tgts = DefaultDataSetInfo().GetTargetInfos();
   std::vector<VariableInfo>& spec = DefaultDataSetInfo().GetSpectatorInfos();

   if (fATreeEvent.size()==0) fATreeEvent.resize(vars.size()+tgts.size()+spec.size());
   // add variables
   for (UInt_t ivar=0; ivar<vars.size(); ivar++) {
      TString vname = vars[ivar].GetExpression();
      assignTree->Branch( vname, &fATreeEvent[ivar], vname + "/F" );
   }
   // add targets<commit_msg>[tmva] release VarTransformHandler also in case of error<commit_after>// @(#)root/tmva $Id$
// Author: <NAME>
// Mentors: <NAME>, <NAME>
//NOTE: Based on TMVA::Factory

/**********************************************************************************
 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis       *
 * Package: TMVA                                                                  *
 * Class  : DataLoader                                                            *
 * Web    : http://tmva.sourceforge.net                                           *
 *                                                                                *
 * Description:                                                                   *
 *      This is a class to load datasets into every booked method                 *
 *                                                                                *
 * Authors (alphabetical):                                                        *
 *      <NAME> <<EMAIL>> - CERN, Switzerland               *
 *      <NAME> <<EMAIL>>  - ITM/UdeA, Colombia                   *
 *      <NAME><<EMAIL>> - CERN, Switzerland                *
 *                                                                                *
 * Copyright (c) 2005-2015:                                                       *
 *      CERN, Switzerland                                                         *
 *      ITM/UdeA, Colombia                                                        *
 *                                                                                *
 * Redistribution and use in source and binary forms, with or without             *
 * modification, are permitted according to the terms listed in LICENSE           *
 * (http://tmva.sourceforge.net/LICENSE)                                          *
 **********************************************************************************/


/*! \class TMVA::DataLoader
\ingroup TMVA

*/

#include "TTree.h"
#include "TH2.h"
#include "TMatrixD.h"

#include "TMVA/DataLoader.h"
#include "TMVA/Config.h"
#include "TMVA/CvSplit.h"
#include "TMVA/Tools.h"
#include "TMVA/IMethod.h"
#include "TMVA/MethodBase.h"
#include "TMVA/DataInputHandler.h"
#include "TMVA/DataSetManager.h"
#include "TMVA/DataSetInfo.h"
#include "TMVA/MethodBoost.h"
#include "TMVA/MethodCategory.h"

#include "TMVA/VariableInfo.h"
#include "TMVA/VariableIdentityTransform.h"
#include "TMVA/VariableDecorrTransform.h"
#include "TMVA/VariablePCATransform.h"
#include "TMVA/VariableGaussTransform.h"
#include "TMVA/VariableNormalizeTransform.h"
#include "TMVA/VarTransformHandler.h"

ClassImp(TMVA::DataLoader);

////////////////////////////////////////////////////////////////////////////////
/*** Create a data loader
 \param[in] thedlName  name of DataLoader object. This name will be used as the
                       top directory name where the training results
                        (weights, i.e .XML and .C files) will be stored.
                       The results will be stored by default in the `theDlName/weights`
                       directory and relative to the current directory. If the directory is not existing,
                       a new one will be created automatically.
                       For using a different location (i.e. a different path to the current directory) one
                       can set an absolute path location in `TMVA::gConfig()::GetIONames().fWeightFileDirPrefix`
                       For example, by setting
~~~~~~~~~~~~~~~{.cpp}
   TMVA::gConfig()::GetIONames().fWeightFileDirPrefix = "/tmp";
   TMVA::gConfig()::GetIONames().fWeightFileDir = "myTrainigResults";
~~~~~~~~~~~~~~~
                       The training results will be stored in the `/tmp/thedlName/myTrainingResults`
                       directory.
**/

TMVA::DataLoader::DataLoader( TString thedlName)
: Configurable( ),
   fDataSetManager       ( NULL ), //DSMTEST
   fDataInputHandler     ( new DataInputHandler ),
   fTransformations      ( "I" ),
   fVerbose              ( kFALSE ),
   fDataAssignType       ( kAssignEvents ),
   fATreeEvent           (0)
{
   fDataSetManager = new DataSetManager( *fDataInputHandler ); // DSMTEST
   SetName(thedlName.Data());
   fLogger->SetSource("DataLoader");
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataLoader::~DataLoader( void )
{
   // destructor

   std::vector<TMVA::VariableTransformBase*>::iterator trfIt = fDefaultTrfs.begin();
   for (;trfIt != fDefaultTrfs.end(); ++trfIt) delete (*trfIt);

   delete fDataInputHandler;

   // destroy singletons
   //   DataSetManager::DestroyInstance(); // DSMTEST replaced by following line
   delete fDataSetManager; // DSMTEST

   // problem with call of REGISTER_METHOD macro ...
   //   ClassifierDataLoader::DestroyInstance();
   //   Types::DestroyInstance();
   //Tools::DestroyInstance();
   //Config::DestroyInstance();
}


////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::AddDataSet( DataSetInfo &dsi )
{
   return fDataSetManager->AddDataSetInfo(dsi); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::AddDataSet( const TString& dsiName )
{
   DataSetInfo* dsi = fDataSetManager->GetDataSetInfo(dsiName); // DSMTEST

   if (dsi!=0) return *dsi;

   return fDataSetManager->AddDataSetInfo(*(new DataSetInfo(dsiName))); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////

TMVA::DataSetInfo& TMVA::DataLoader::GetDataSetInfo()
{
   return DefaultDataSetInfo(); // DSMTEST
}

////////////////////////////////////////////////////////////////////////////////
/// Transforms the variables and return a new DataLoader with the transformed
/// variables

TMVA::DataLoader* TMVA::DataLoader::VarTransform(TString trafoDefinition)
{
   TString trOptions = "0";
   TString trName = "None";
   if (trafoDefinition.Contains("(")) {

      // contains transformation parameters
      Ssiz_t parStart = trafoDefinition.Index( "(" );
      Ssiz_t parLen   = trafoDefinition.Index( ")", parStart )-parStart+1;

      trName = trafoDefinition(0,parStart);
      trOptions = trafoDefinition(parStart,parLen);
      trOptions.Remove(parLen-1,1);
      trOptions.Remove(0,1);
   }
   else
      trName = trafoDefinition;

   VarTransformHandler* handler = new VarTransformHandler(this);
   // variance threshold variable transformation
   if (trName == "VT") {

      // find threshold value from given input
      Double_t threshold = 0.0;
      if (!trOptions.IsFloat()){
         Log() << kFATAL << " VT transformation must be passed a floating threshold value" << Endl;
         delete handler;
         return this;
      }
      else
         threshold =  trOptions.Atof();
      TMVA::DataLoader *transformedLoader = handler->VarianceThreshold(threshold);
      delete handler;
      return transformedLoader;
   }
   else {
      delete handler;
      Log() << kFATAL << "Incorrect transformation string provided, please check" << Endl;
   }
   Log() << kINFO << "No transformation applied, returning original loader" << Endl;
   return this;
}

////////////////////////////////////////////////////////////////////////////////
// the next functions are to assign events directly

////////////////////////////////////////////////////////////////////////////////
/// create the data assignment tree (for event-wise data assignment by user)

TTree* TMVA::DataLoader::CreateEventAssignTrees( const TString& name )
{
   TTree * assignTree = new TTree( name, name );
   assignTree->SetDirectory(0);
   assignTree->Branch( "type",   &fATreeType,   "ATreeType/I" );
   assignTree->Branch( "weight", &fATreeWeight, "ATreeWeight/F" );

   std::vector<VariableInfo>& vars = DefaultDataSetInfo().GetVariableInfos();
   std::vector<VariableInfo>& tgts = DefaultDataSetInfo().GetTargetInfos();
   std::vector<VariableInfo>& spec = DefaultDataSetInfo().GetSpectatorInfos();

   if (fATreeEvent.size()==0) fATreeEvent.resize(vars.size()+tgts.size()+spec.size());
   // add variables
   for (UInt_t ivar=0; ivar<vars.size(); ivar++) {
      TString vname = vars[ivar].GetExpression();
      assignTree->Branch( vname, &fATreeEvent[ivar], vname + "/F" );
   }
   // add targets
{'fraction_non_alphanumeric': 0.20126650337535099, 'fraction_numerical': 0.00274807336161061, 'mean_word_length': 1.7078615334843092, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3802577', 'n_tokens_mistral': 4220, 'n_tokens_neox': 4124, 'n_words': 1010}