<commit_before>﻿// This file is part of the AliceVision project.
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

#include <aliceVision/config.hpp>
#include <aliceVision/alicevision_omp.hpp>
#include <aliceVision/image/all.hpp>
#include <aliceVision/sfm/sfm.hpp>
#include <aliceVision/feature/imageDescriberCommon.hpp>
#include <aliceVision/feature/feature.hpp>
#include <aliceVision/system/MemoryInfo.hpp>
#include <aliceVision/system/Timer.hpp>
#include <aliceVision/system/Logger.hpp>
#include <aliceVision/system/cmdline.hpp>

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/progress.hpp>

#include <string>
#include <fstream>
#include <sstream>
#include <iostream>
#include <functional>
#include <memory>
#include <limits>

using namespace aliceVision;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

class FeatureExtractor
{
  struct ViewJob
  {
    const sfm::View& view;
    std::size_t memoryConsuption = 0;
    std::string outputBasename;
    std::vector<std::size_t> cpuImageDescriberIndexes;
    std::vector<std::size_t> gpuImageDescriberIndexes;

    ViewJob(const sfm::View& view,
            const std::string& outputFolder)
      : view(view)
      , outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string())
    {}

    bool useGPU() const
    {
      return !gpuImageDescriberIndexes.empty();
    }

    bool useCPU() const
    {
      return !cpuImageDescriberIndexes.empty();
    }

    std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const
    {
      return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".feat";
    }

    std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const
    {
      return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".desc";
    }

    void setImageDescribers(const std::vector<std::shared_ptr<feature::ImageDescriber>>& imageDescribers)
    {
      for(std::size_t i = 0; i < imageDescribers.size(); ++i)
      {
        const std::shared_ptr<feature::ImageDescriber>& imageDescriber = imageDescribers.at(i);
        feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType();

        if(fs::exists(getFeaturesPath(imageDescriberType)) &&
           fs::exists(getDescriptorPath(imageDescriberType)))
          continue;

        memoryConsuption += imageDescriber->getMemoryConsumption(view.getWidth(), view.getHeight());

        if(imageDescriber->useCuda())
          gpuImageDescriberIndexes.push_back(i);
        else
          cpuImageDescriberIndexes.push_back(i);
      }
    }
  };

public:

  explicit FeatureExtractor(const sfm::SfMData& sfmData)
    : _sfmData(sfmData)
  {}

  void setRange(int rangeStart, int rangeSize)
  {
    _rangeStart = rangeStart;
    _rangeSize = rangeSize;
  }

  void setOutputFolder(const std::string& folder)
  {
    _outputFolder = folder;
  }

  void addImageDescriber(std::shared_ptr<feature::ImageDescriber>& imageDescriber)
  {
    _imageDescribers.push_back(imageDescriber);
  }

  void process()
  {
    // iteration on each view in the range in order
    // to prepare viewJob stack
    sfm::Views::const_iterator itViewBegin = _sfmData.GetViews().begin();
    sfm::Views::const_iterator itViewEnd = _sfmData.GetViews().begin();

    if(_rangeStart != -1)
    {
      std::advance(itViewBegin, _rangeStart);
      std::advance(itViewEnd, _rangeStart + _rangeSize);
    }

    std::size_t jobMaxMemoryConsuption = 0;

    for(auto it = itViewBegin; it != itViewEnd; ++it)
    {
      const sfm::View& view = *(it->second.get());
      ViewJob viewJob(view, _outputFolder);

      viewJob.setImageDescribers(_imageDescribers);
      jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption);

      if(viewJob.useCPU())
        _cpuJobs.push_back(viewJob);

      if(viewJob.useGPU())
        _gpuJobs.push_back(viewJob);
    }

    if(!_cpuJobs.empty())
    {
      system::MemoryInfo memoryInformation = system::getMemoryInfo();

      ALICEVISION_LOG_DEBUG("Job max memory consumption: " << jobMaxMemoryConsuption << " B");
      ALICEVISION_LOG_DEBUG("Memory information: " << std::endl <<memoryInformation);

      if(jobMaxMemoryConsuption == 0)
        throw std::runtime_error("Can't compute feature extraction job max memory consuption.");

      std::size_t nbThreads =  (0.9 * memoryInformation.freeRam) / jobMaxMemoryConsuption;

      if(memoryInformation.freeRam == 0)
      {
        ALICEVISION_LOG_WARNING("Can't find available system memory, this can be due to OS limitations.\n"
                                "Use only one thread for CPU feature extraction.");
        nbThreads = 1;
      }

      // nbThreads should not be higher than the core number
      nbThreads = std::min(static_cast<std::size_t>(omp_get_num_procs()), nbThreads);

      ALICEVISION_LOG_DEBUG("# threads for extraction: " << nbThreads);
      omp_set_nested(1);

#pragma omp parallel for num_threads(nbThreads)<commit_msg>[software] `featureExtraction` fix : unset range & threads<commit_after>﻿// This file is part of the AliceVision project.
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

#include <aliceVision/config.hpp>
#include <aliceVision/alicevision_omp.hpp>
#include <aliceVision/image/all.hpp>
#include <aliceVision/sfm/sfm.hpp>
#include <aliceVision/feature/imageDescriberCommon.hpp>
#include <aliceVision/feature/feature.hpp>
#include <aliceVision/system/MemoryInfo.hpp>
#include <aliceVision/system/Timer.hpp>
#include <aliceVision/system/Logger.hpp>
#include <aliceVision/system/cmdline.hpp>

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/progress.hpp>

#include <string>
#include <fstream>
#include <sstream>
#include <iostream>
#include <functional>
#include <memory>
#include <limits>

using namespace aliceVision;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

class FeatureExtractor
{
  struct ViewJob
  {
    const sfm::View& view;
    std::size_t memoryConsuption = 0;
    std::string outputBasename;
    std::vector<std::size_t> cpuImageDescriberIndexes;
    std::vector<std::size_t> gpuImageDescriberIndexes;

    ViewJob(const sfm::View& view,
            const std::string& outputFolder)
      : view(view)
      , outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string())
    {}

    bool useGPU() const
    {
      return !gpuImageDescriberIndexes.empty();
    }

    bool useCPU() const
    {
      return !cpuImageDescriberIndexes.empty();
    }

    std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const
    {
      return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".feat";
    }

    std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const
    {
      return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".desc";
    }

    void setImageDescribers(const std::vector<std::shared_ptr<feature::ImageDescriber>>& imageDescribers)
    {
      for(std::size_t i = 0; i < imageDescribers.size(); ++i)
      {
        const std::shared_ptr<feature::ImageDescriber>& imageDescriber = imageDescribers.at(i);
        feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType();

        if(fs::exists(getFeaturesPath(imageDescriberType)) &&
           fs::exists(getDescriptorPath(imageDescriberType)))
          continue;

        memoryConsuption += imageDescriber->getMemoryConsumption(view.getWidth(), view.getHeight());

        if(imageDescriber->useCuda())
          gpuImageDescriberIndexes.push_back(i);
        else
          cpuImageDescriberIndexes.push_back(i);
      }
    }
  };

public:

  explicit FeatureExtractor(const sfm::SfMData& sfmData)
    : _sfmData(sfmData)
  {}

  void setRange(int rangeStart, int rangeSize)
  {
    _rangeStart = rangeStart;
    _rangeSize = rangeSize;
  }

  void setOutputFolder(const std::string& folder)
  {
    _outputFolder = folder;
  }

  void addImageDescriber(std::shared_ptr<feature::ImageDescriber>& imageDescriber)
  {
    _imageDescribers.push_back(imageDescriber);
  }

  void process()
  {
    // iteration on each view in the range in order
    // to prepare viewJob stack
    sfm::Views::const_iterator itViewBegin = _sfmData.GetViews().begin();
    sfm::Views::const_iterator itViewEnd = _sfmData.GetViews().end();

    if(_rangeStart != -1)
    {
      std::advance(itViewBegin, _rangeStart);
      itViewEnd = itViewBegin;
      std::advance(itViewEnd, _rangeSize);
    }

    std::size_t jobMaxMemoryConsuption = 0;

    for(auto it = itViewBegin; it != itViewEnd; ++it)
    {
      const sfm::View& view = *(it->second.get());
      ViewJob viewJob(view, _outputFolder);

      viewJob.setImageDescribers(_imageDescribers);
      jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption);

      if(viewJob.useCPU())
        _cpuJobs.push_back(viewJob);

      if(viewJob.useGPU())
        _gpuJobs.push_back(viewJob);
    }

    if(!_cpuJobs.empty())
    {
      system::MemoryInfo memoryInformation = system::getMemoryInfo();

      ALICEVISION_LOG_DEBUG("Job max memory consumption: " << jobMaxMemoryConsuption << " B");
      ALICEVISION_LOG_DEBUG("Memory information: " << std::endl <<memoryInformation);

      if(jobMaxMemoryConsuption == 0)
        throw std::runtime_error("Can't compute feature extraction job max memory consuption.");

      std::size_t nbThreads =  (0.9 * memoryInformation.freeRam) / jobMaxMemoryConsuption;

      if(memoryInformation.freeRam == 0)
      {
        ALICEVISION_LOG_WARNING("Can't find available system memory, this can be due to OS limitations.\n"
                                "Use only one thread for CPU feature extraction.");
        nbThreads = 1;
      }

      // nbThreads should not be higher than the core number
      nbThreads = std::min(static_cast<std::size_t>(omp_get_num_procs()), nbThreads);

      // nbThreads should not be higher than the job number
      nbThreads = std::min(_cpuJobs.size(), nbThreads);

      ALICEVISION_LOG_DEBUG("# threads for extraction: " << nbThreads);
      omp_set_nested(1);

#pragma omp parallel for num_threads(nbThreads)
{'fraction_non_alphanumeric': 0.11881464914733017, 'fraction_numerical': 0.002609262883235486, 'mean_word_length': 3.849525530953457, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 65, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5531', 'n_tokens_mistral': 3528, 'n_tokens_neox': 3360, 'n_words': 665}