<commit_before>#include "Run.h"
#include "InputContainer.h"
#include "VarConf.h"

Run::Run(std::string iTag) : mRunName(iTag) {
   Options runOptions = loadRunOptions(iTag);
   init(runOptions);
}
Run::Run(const Options& iOptions) {
   init(iOptions);
}
void Run::init(const Options& iOptions) {
   mRunOptions = iOptions;

   // Create input service
   mInputContainer = new InputContainer(Options(""));

   // Create a default data object
   // Each configuration needs its own data object, since a configuration can choose its own
   // input sources, downscaler, etc. However we to reuse the downscaler, etc for those
   // configurations that do not need customization. Therefore create a default data object that can
   // reused if needed.
   Options dataOptions0;
   dataOptions0.addOption("runName", mRunName);
   Options::copyOption("inputs", mRunOptions, dataOptions0);
   Options::copyOption("qcs",    mRunOptions, dataOptions0);
   if(!dataOptions0.hasValues("inputs")) {
      std::stringstream ss;
      ss << "Cannot initialize data object. 'inputs' option not provided for run '"
         << mRunName << "'";
      Global::logger->write(ss.str(), Logger::error);
   }
   mDefaultData = new Data(dataOptions0, mInputContainer);

   // Get variable/configurations
   std::map<std::string, std::vector<std::string> > varConfs;
   std::map<std::string, std::vector<std::string> > varMetrics;
   loadVarConfs(mRunOptions, varConfs, varMetrics, mVariables);

   // Loop over all variables
   for(int v = 0; v < mVariables.size(); v++) {
      std::string variable = mVariables[v];

      // Loop over all configurations
      std::vector<std::string> configurations = varConfs[variable];
      for(int c = 0; c < configurations.size(); c++) {
         // Set up options for configuration
         Options configOptions;
         Configuration::getOptions(configurations[c], configOptions);

         ////////////////////////////////////////
         // Pass run options to configurations //
         ////////////////////////////////////////
         // Pass down which output offsets we are producing for (parameter estimation needs it)
         Options::copyOption("offsets", mRunOptions, configOptions);
         Options::copyOption("spreader", mRunOptions, configOptions);
         if(!configOptions.hasValue("parameterIo")) {
            Options::copyOption("parameterIo", mRunOptions, configOptions);
         }
         if(!configOptions.hasValues("inputs")) {
            Options::copyOption("inputs", mRunOptions, configOptions);
         }
         if(!configOptions.hasValue("numOffsetsSpreadObs")) {
            Options::copyOption("numOffsetsSpreadObs", mRunOptions, configOptions);
         }
         Options::copyOption("numDaysParameterSearch", mRunOptions, configOptions);
         if(!configOptions.hasValue("pooler")) {
            // Set up how to choose which obs to use in parameter estimation
            Options::copyOption("pooler", mRunOptions, configOptions);
         }

         ////////////////////////////////////////
         // Create data for this configuration //
         ////////////////////////////////////////
         Data* data;
         if(configOptions.hasValues("inputs") || configOptions.hasValues("qcs") || configOptions.hasValue("downscaler")) {
            // Create custom data object. NOTE: If any of options are cutomized by configuration,
            // then the downscaler and qcs cannot be reused, since their cached values
            // may be incorrect

            // Initialize with default, then overwrite
            Options dataOptions = dataOptions0;
            dataOptions.addOption("runName", mRunName);
            Options::copyOption("inputs", configOptions, dataOptions);
            // Only copy if it is there
            Options::copyOption("qcs",    configOptions, dataOptions);
            Options::copyOption("downscaler", configOptions, dataOptions);
            if(!dataOptions.hasValues("inputs")) {
               std::stringstream ss;
               ss << "Cannot initialize data object. 'inputs' neither provided for run '"
                  << mRunName << "' nor for configuration '"
                  << configurations[c] << "'.";
               Global::logger->write(ss.str(), Logger::error);
            }
            data = new Data(dataOptions, mInputContainer);
            mConfigDatas.push_back(data);
         }
         else {
            // No need to create a custom data object, use the default one
            data = mDefaultData;
         }
         Configuration* conf = Configuration::getScheme(configOptions, *data);
         mVarConfs[variable].push_back(conf);
      }

      // Metrics
      std::vector<std::string> metrics = varMetrics[variable];
      for(int m = 0; m < metrics.size(); m++) {
         Metric* metric = Metric::getScheme(metrics[m], *mDefaultData);
         mMetrics[variable].push_back(metric);
      }
   }

   // Outputs
   std::vector<std::string> outputsString;
   //! What output schemes should the forecasts we written to?
   mRunOptions.getValues("outputs", outputsString);
   for(int i = 0; i < outputsString.size(); i++) {
      Output* output = Output::getScheme(outputsString[i], *mDefaultData);
      mOutputs.push_back(output);
   }

   // Debug
   int debug = mDefaultDebugLevel;
   //! How much debug and status messages should be shown? 0 very littly, 1000 all. Default to 50 (a
   //! reasonable amount).
   mRunOptions.getValue("debug", debug);
   Global::logger->setMaxLevel((Logger::Level) debug);

   //////////////
   // Location //
   //////////////
   std::vector<std::string> locationTags;
   //! Which locations should forecasts be produced for? Give a list of datasets. If no list given
   //! the locations from the observation dataset will be used.
   if(mRunOptions.getValues("locations", locationTags)) {
      for(int i = 0; i < locationTags.size(); i++) {
         Input* input = mInputContainer->getInput(locationTags[i]);
         if(input != NULL) {
            std::vector<Location> locations = input->getLocations();
            for(int k = 0; k < locations.size(); k++) {
               mLocations.push_back(locations[k]);
            }
         }
         else {
            // This code should not execute, because Input should never return NULL
            std::stringstream ss;
            ss << "Error when reading 'locations' option in " << getName() 
               << " run. Input '" << locationTags[i] << "' does not exist.";
            Global::logger->write(ss.str(), Logger::error);
         }
      }
   }
   else {
      // Assume that we want to forecast for locations where have obs
      Input* input = mDefaultData->getObsInput();
      if(input != NULL) {
         mLocations = input->getLocations();<commit_msg>If no obs dataset specified, use forecast locations<commit_after>#include "Run.h"
#include "InputContainer.h"
#include "VarConf.h"

Run::Run(std::string iTag) : mRunName(iTag) {
   Options runOptions = loadRunOptions(iTag);
   init(runOptions);
}
Run::Run(const Options& iOptions) {
   init(iOptions);
}
void Run::init(const Options& iOptions) {
   mRunOptions = iOptions;

   // Create input service
   mInputContainer = new InputContainer(Options(""));

   // Create a default data object
   // Each configuration needs its own data object, since a configuration can choose its own
   // input sources, downscaler, etc. However we to reuse the downscaler, etc for those
   // configurations that do not need customization. Therefore create a default data object that can
   // reused if needed.
   Options dataOptions0;
   dataOptions0.addOption("runName", mRunName);
   Options::copyOption("inputs", mRunOptions, dataOptions0);
   Options::copyOption("qcs",    mRunOptions, dataOptions0);
   if(!dataOptions0.hasValues("inputs")) {
      std::stringstream ss;
      ss << "Cannot initialize data object. 'inputs' option not provided for run '"
         << mRunName << "'";
      Global::logger->write(ss.str(), Logger::error);
   }
   mDefaultData = new Data(dataOptions0, mInputContainer);

   // Get variable/configurations
   std::map<std::string, std::vector<std::string> > varConfs;
   std::map<std::string, std::vector<std::string> > varMetrics;
   loadVarConfs(mRunOptions, varConfs, varMetrics, mVariables);

   // Loop over all variables
   for(int v = 0; v < mVariables.size(); v++) {
      std::string variable = mVariables[v];

      // Loop over all configurations
      std::vector<std::string> configurations = varConfs[variable];
      for(int c = 0; c < configurations.size(); c++) {
         // Set up options for configuration
         Options configOptions;
         Configuration::getOptions(configurations[c], configOptions);

         ////////////////////////////////////////
         // Pass run options to configurations //
         ////////////////////////////////////////
         // Pass down which output offsets we are producing for (parameter estimation needs it)
         Options::copyOption("offsets", mRunOptions, configOptions);
         Options::copyOption("spreader", mRunOptions, configOptions);
         if(!configOptions.hasValue("parameterIo")) {
            Options::copyOption("parameterIo", mRunOptions, configOptions);
         }
         if(!configOptions.hasValues("inputs")) {
            Options::copyOption("inputs", mRunOptions, configOptions);
         }
         if(!configOptions.hasValue("numOffsetsSpreadObs")) {
            Options::copyOption("numOffsetsSpreadObs", mRunOptions, configOptions);
         }
         Options::copyOption("numDaysParameterSearch", mRunOptions, configOptions);
         if(!configOptions.hasValue("pooler")) {
            // Set up how to choose which obs to use in parameter estimation
            Options::copyOption("pooler", mRunOptions, configOptions);
         }

         ////////////////////////////////////////
         // Create data for this configuration //
         ////////////////////////////////////////
         Data* data;
         if(configOptions.hasValues("inputs") || configOptions.hasValues("qcs") || configOptions.hasValue("downscaler")) {
            // Create custom data object. NOTE: If any of options are cutomized by configuration,
            // then the downscaler and qcs cannot be reused, since their cached values
            // may be incorrect

            // Initialize with default, then overwrite
            Options dataOptions = dataOptions0;
            dataOptions.addOption("runName", mRunName);
            Options::copyOption("inputs", configOptions, dataOptions);
            // Only copy if it is there
            Options::copyOption("qcs",    configOptions, dataOptions);
            Options::copyOption("downscaler", configOptions, dataOptions);
            if(!dataOptions.hasValues("inputs")) {
               std::stringstream ss;
               ss << "Cannot initialize data object. 'inputs' neither provided for run '"
                  << mRunName << "' nor for configuration '"
                  << configurations[c] << "'.";
               Global::logger->write(ss.str(), Logger::error);
            }
            data = new Data(dataOptions, mInputContainer);
            mConfigDatas.push_back(data);
         }
         else {
            // No need to create a custom data object, use the default one
            data = mDefaultData;
         }
         Configuration* conf = Configuration::getScheme(configOptions, *data);
         mVarConfs[variable].push_back(conf);
      }

      // Metrics
      std::vector<std::string> metrics = varMetrics[variable];
      for(int m = 0; m < metrics.size(); m++) {
         Metric* metric = Metric::getScheme(metrics[m], *mDefaultData);
         mMetrics[variable].push_back(metric);
      }
   }

   // Outputs
   std::vector<std::string> outputsString;
   //! What output schemes should the forecasts we written to?
   mRunOptions.getValues("outputs", outputsString);
   for(int i = 0; i < outputsString.size(); i++) {
      Output* output = Output::getScheme(outputsString[i], *mDefaultData);
      mOutputs.push_back(output);
   }

   // Debug
   int debug = mDefaultDebugLevel;
   //! How much debug and status messages should be shown? 0 very littly, 1000 all. Default to 50 (a
   //! reasonable amount).
   mRunOptions.getValue("debug", debug);
   Global::logger->setMaxLevel((Logger::Level) debug);

   //////////////
   // Location //
   //////////////
   std::vector<std::string> locationTags;
   //! Which locations should forecasts be produced for? Give a list of datasets. If no list given
   //! the locations from the observation dataset will be used.
   if(mRunOptions.getValues("locations", locationTags)) {
      for(int i = 0; i < locationTags.size(); i++) {
         Input* input = mInputContainer->getInput(locationTags[i]);
         if(input != NULL) {
            std::vector<Location> locations = input->getLocations();
            for(int k = 0; k < locations.size(); k++) {
               mLocations.push_back(locations[k]);
            }
         }
         else {
            // This code should not execute, because if the input scheme doesn't exist, the program 
            // will already have been aborted. This is to protect against future changes.
            std::stringstream ss;
            ss << "Error when reading 'locations' option in " << getName() 
               << " run. Input '" << locationTags[i] << "' does not exist.";
            Global::logger->write(ss.str(), Logger::error);
         }
      }
   }
   else {
      // Assume that we want to forecast for locations where have obs
      Input* input = mDefaultData->getObsInput();
      // Otherwise assume that we want to forecast for locations where we have forecasts
      if(input == NULL) {
         input = mDefaultData->getInput();
         std::stringstream ss;
         ss << "No observation dataset specified. Producing forecasts for each location in the main forecast dataset.";
         Global::logger->write(ss.str(), Logger::warning);
      }
      if(input != NULL) {
         mLocations = input->getLocations();
{'fraction_non_alphanumeric': 0.1432311938291699, 'fraction_numerical': 0.0028306560045290494, 'mean_word_length': 2.4300970873786407, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4278863', 'n_tokens_mistral': 3665, 'n_tokens_neox': 3574, 'n_words': 1037}