<commit_before>// -----------------------------------------------------------------------------
//
// Copyright (C) The BioDynaMo Project.
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
//
// See the LICENSE file distributed with this work for details.
// See the NOTICE file distributed with this work for additional information
// regarding copyright ownership.
//
// -----------------------------------------------------------------------------

#include "core/simulation.h"

#include <cpptoml/cpptoml.h>
#include <omp.h>
#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <fstream>
#include <memory>
#include <ostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include "core/environment/environment.h"
#include "core/environment/uniform_grid_environment.h"
#include "core/execution_context/in_place_exec_ctxt.h"
#include "core/gpu/gpu_helper.h"
#include "core/param/command_line_options.h"
#include "core/param/param.h"
#include "core/resource_manager.h"
#include "core/scheduler.h"
#include "core/sim_object/so_uid_generator.h"
#include "core/util/io.h"
#include "core/util/log.h"
#include "core/util/string.h"
#include "core/util/thread_info.h"
#include "core/util/timing.h"
#include "core/visualization/root/adaptor.h"
#include "version.h"

#include <TEnv.h>

namespace bdm {

/// Implementation for `Simulation`:
/// It must be separate to avoid circular dependencies.

std::atomic<uint64_t> Simulation::counter_;

Simulation* Simulation::active_ = nullptr;

Simulation* Simulation::GetActive() { return active_; }

Simulation::Simulation(TRootIOCtor* p) {}

Simulation::Simulation(int argc, const char** argv,
                       const std::string& config_file)
    : Simulation(
          argc, argv, [](auto* param) {}, config_file) {}

Simulation::Simulation(const std::string& simulation_name,
                       const std::string& config_file)
    : Simulation(
          simulation_name, [](auto* param) {}, config_file) {}

Simulation::Simulation(CommandLineOptions* clo,
                       const std::string& config_file) {
  Initialize(
      clo, [](auto* param) {}, config_file);
}

Simulation::Simulation(CommandLineOptions* clo,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  Initialize(clo, set_param, config_file);
}

Simulation::Simulation(int argc, const char** argv,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  auto options = CommandLineOptions(argc, argv);
  Initialize(&options, set_param, config_file);
}

Simulation::Simulation(const std::string& simulation_name,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  const char* argv[1] = {simulation_name.c_str()};
  auto options = CommandLineOptions(1, argv);
  Initialize(&options, set_param, config_file);
}

void Simulation::Restore(Simulation&& restored) {
  // random_
  if (random_.size() != restored.random_.size()) {
    Log::Warning("Simulation", "The restore file (", param_->restore_file_,
                 ") was run with a different number of threads. Can't restore "
                 "complete random number generator state.");
    uint64_t min = std::min(random_.size(), restored.random_.size());
    for (uint64_t i = 0; i < min; i++) {
      *(random_[i]) = *(restored.random_[i]);
    }
  } else {
    for (uint64_t i = 0; i < random_.size(); i++) {
      *(random_[i]) = *(restored.random_[i]);
    }
  }

  // param and rm
  param_->Restore(std::move(*restored.param_));
  restored.param_ = nullptr;
  *rm_ = std::move(*restored.rm_);
  restored.rm_ = nullptr;

  // name_ and unique_name_
  InitializeUniqueName(restored.name_);
  InitializeOutputDir();
}

std::ostream& operator<<(std::ostream& os, Simulation& sim) {
  std::vector<std::string> dg_names;
  std::vector<int> dg_resolutions;
  std::vector<std::array<int32_t, 3>> dg_dimensions;

  sim.rm_->ApplyOnAllDiffusionGrids([&](auto* dg) {
    dg_names.push_back(dg->GetSubstanceName());
    dg_resolutions.push_back(dg->GetResolution());
    dg_dimensions.push_back(dg->GetGridSize());
  });

  os << std::endl;

  os << "***********************************************" << std::endl;
  os << "***********************************************" << std::endl;
  os << "\033[1mSimulation Metadata:\033[0m" << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mGeneral\033[0m" << std::endl;
  if (sim.command_line_parameter_str_ != "") {
    os << "Command\t\t\t\t: " << sim.command_line_parameter_str_ << std::endl;
  }
  os << "Simulation name\t\t\t: " << sim.GetUniqueName() << std::endl;
  os << "Number of iterations executed\t: "
     << sim.scheduler_->GetSimulatedSteps() << std::endl;
  os << "Number of simulation objects\t: " << sim.rm_->GetNumSimObjects()
     << std::endl;

  if (dg_names.size() != 0) {
    os << "Diffusion grids" << std::endl;
    for (size_t i = 0; i < dg_names.size(); ++i) {
      os << "  " << dg_names[i] << ":" << std::endl;
      auto& dim = dg_dimensions[i];
      os << "\t"
         << "Resolution\t\t: " << dg_resolutions[i] << std::endl;
      os << "\t"
         << "Size\t\t\t: " << dim[0] << " x " << dim[1] << " x " << dim[2]
         << std::endl;
    }
  }

  os << "Output directory\t\t: " << sim.GetOutputDir() << std::endl;
  os << "  size\t\t\t\t: "
     << gSystem->GetFromPipe(
            Concat("du -sh ", sim.GetOutputDir(), " | cut -f1").c_str())
     << std::endl;
  os << "BioDynaMo version:\t\t: " << Version::String() << std::endl;
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << *(sim.scheduler_->GetOpTimes()) << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mThread Info\033[0m" << std::endl;
  os << *ThreadInfo::GetInstance();
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << *(sim.rm_);
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mParameters\033[0m" << std::endl;
  os << sim.param_->ToJsonString();
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << "***********************************************" << std::endl;

  return os;
}

Simulation::~Simulation() {
  if (param_ != nullptr && param_->statistics_) {
    std::cout << *this << std::endl;
  }

  if (mem_mgr_) {
    mem_mgr_->SetIgnoreDelete(true);
  }
  Simulation* tmp = nullptr;
  if (active_ != this) {
    tmp = active_;
  }
  active_ = this;

  delete rm_;
  delete environment_;
  delete scheduler_;
  if (so_uid_generator_ != nullptr) {
    delete so_uid_generator_;
  }
  delete param_;
  for (auto* r : random_) {
    delete r;
  }
  for (auto* ectxt : exec_ctxt_) {
    delete ectxt;
  }
  if (mem_mgr_) {
    delete mem_mgr_;
  }<commit_msg>Write simulation metadata to file<commit_after>// -----------------------------------------------------------------------------
//
// Copyright (C) The BioDynaMo Project.
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
//
// See the LICENSE file distributed with this work for details.
// See the NOTICE file distributed with this work for additional information
// regarding copyright ownership.
//
// -----------------------------------------------------------------------------

#include "core/simulation.h"

#include <cpptoml/cpptoml.h>
#include <omp.h>
#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <fstream>
#include <memory>
#include <ostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include "core/environment/environment.h"
#include "core/environment/uniform_grid_environment.h"
#include "core/execution_context/in_place_exec_ctxt.h"
#include "core/gpu/gpu_helper.h"
#include "core/param/command_line_options.h"
#include "core/param/param.h"
#include "core/resource_manager.h"
#include "core/scheduler.h"
#include "core/sim_object/so_uid_generator.h"
#include "core/util/io.h"
#include "core/util/log.h"
#include "core/util/string.h"
#include "core/util/thread_info.h"
#include "core/util/timing.h"
#include "core/visualization/root/adaptor.h"
#include "version.h"

#include <TEnv.h>

namespace bdm {

/// Implementation for `Simulation`:
/// It must be separate to avoid circular dependencies.

std::atomic<uint64_t> Simulation::counter_;

Simulation* Simulation::active_ = nullptr;

Simulation* Simulation::GetActive() { return active_; }

Simulation::Simulation(TRootIOCtor* p) {}

Simulation::Simulation(int argc, const char** argv,
                       const std::string& config_file)
    : Simulation(
          argc, argv, [](auto* param) {}, config_file) {}

Simulation::Simulation(const std::string& simulation_name,
                       const std::string& config_file)
    : Simulation(
          simulation_name, [](auto* param) {}, config_file) {}

Simulation::Simulation(CommandLineOptions* clo,
                       const std::string& config_file) {
  Initialize(
      clo, [](auto* param) {}, config_file);
}

Simulation::Simulation(CommandLineOptions* clo,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  Initialize(clo, set_param, config_file);
}

Simulation::Simulation(int argc, const char** argv,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  auto options = CommandLineOptions(argc, argv);
  Initialize(&options, set_param, config_file);
}

Simulation::Simulation(const std::string& simulation_name,
                       const std::function<void(Param*)>& set_param,
                       const std::string& config_file) {
  const char* argv[1] = {simulation_name.c_str()};
  auto options = CommandLineOptions(1, argv);
  Initialize(&options, set_param, config_file);
}

void Simulation::Restore(Simulation&& restored) {
  // random_
  if (random_.size() != restored.random_.size()) {
    Log::Warning("Simulation", "The restore file (", param_->restore_file_,
                 ") was run with a different number of threads. Can't restore "
                 "complete random number generator state.");
    uint64_t min = std::min(random_.size(), restored.random_.size());
    for (uint64_t i = 0; i < min; i++) {
      *(random_[i]) = *(restored.random_[i]);
    }
  } else {
    for (uint64_t i = 0; i < random_.size(); i++) {
      *(random_[i]) = *(restored.random_[i]);
    }
  }

  // param and rm
  param_->Restore(std::move(*restored.param_));
  restored.param_ = nullptr;
  *rm_ = std::move(*restored.rm_);
  restored.rm_ = nullptr;

  // name_ and unique_name_
  InitializeUniqueName(restored.name_);
  InitializeOutputDir();
}

std::ostream& operator<<(std::ostream& os, Simulation& sim) {
  std::vector<std::string> dg_names;
  std::vector<int> dg_resolutions;
  std::vector<std::array<int32_t, 3>> dg_dimensions;

  sim.rm_->ApplyOnAllDiffusionGrids([&](auto* dg) {
    dg_names.push_back(dg->GetSubstanceName());
    dg_resolutions.push_back(dg->GetResolution());
    dg_dimensions.push_back(dg->GetGridSize());
  });

  os << std::endl;

  os << "***********************************************" << std::endl;
  os << "***********************************************" << std::endl;
  os << "\033[1mSimulation Metadata:\033[0m" << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mGeneral\033[0m" << std::endl;
  if (sim.command_line_parameter_str_ != "") {
    os << "Command\t\t\t\t: " << sim.command_line_parameter_str_ << std::endl;
  }
  os << "Simulation name\t\t\t: " << sim.GetUniqueName() << std::endl;
  os << "Number of iterations executed\t: "
     << sim.scheduler_->GetSimulatedSteps() << std::endl;
  os << "Number of simulation objects\t: " << sim.rm_->GetNumSimObjects()
     << std::endl;

  if (dg_names.size() != 0) {
    os << "Diffusion grids" << std::endl;
    for (size_t i = 0; i < dg_names.size(); ++i) {
      os << "  " << dg_names[i] << ":" << std::endl;
      auto& dim = dg_dimensions[i];
      os << "\t"
         << "Resolution\t\t: " << dg_resolutions[i] << std::endl;
      os << "\t"
         << "Size\t\t\t: " << dim[0] << " x " << dim[1] << " x " << dim[2]
         << std::endl;
    }
  }

  os << "Output directory\t\t: " << sim.GetOutputDir() << std::endl;
  os << "  size\t\t\t\t: "
     << gSystem->GetFromPipe(
            Concat("du -sh ", sim.GetOutputDir(), " | cut -f1").c_str())
     << std::endl;
  os << "BioDynaMo version:\t\t: " << Version::String() << std::endl;
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << *(sim.scheduler_->GetOpTimes()) << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mThread Info\033[0m" << std::endl;
  os << *ThreadInfo::GetInstance();
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << *(sim.rm_);
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << std::endl;
  os << "\033[1mParameters\033[0m" << std::endl;
  os << sim.param_->ToJsonString();
  os << std::endl;
  os << "***********************************************" << std::endl;
  os << "***********************************************" << std::endl;

  return os;
}

Simulation::~Simulation() {
  if (param_ != nullptr && param_->statistics_) {
    std::stringstream sstr;
    sstr << *this << std::endl;
    std::cout << sstr.str() << std::endl;
    // write to file
    std::ofstream ofs(Concat(output_dir_, "/metadata"));
    ofs << sstr.str() << std::endl;
  }

  if (mem_mgr_) {
    mem_mgr_->SetIgnoreDelete(true);
  }
  Simulation* tmp = nullptr;
  if (active_ != this) {
    tmp = active_;
  }
  active_ = this;

  delete rm_;
  delete environment_;
  delete scheduler_;
  if (so_uid_generator_ != nullptr) {
    delete so_uid_generator_;
  }
  delete param_;
  for (auto* r : random_) {
    delete r;
  }
  for (auto* ectxt : exec_ctxt_) {
    delete ectxt;
  }
  if (mem_mgr_) {
    delete mem_mgr_;
  }
{'fraction_non_alphanumeric': 0.24191241571885855, 'fraction_numerical': 0.007491657018320507, 'mean_word_length': 3.3023732786404922, 'pattern_counts': {'":': 2, '<': 387, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '80003', 'n_tokens_mistral': 4809, 'n_tokens_neox': 4728, 'n_words': 1004}