<commit_before>/** @file distributed_dualtree_dfs_dev.h
 *
 *  The generic algorithm template for distributed dual-tree
 *  computation.
 *
 *  @author <NAME> (<EMAIL>)
 */

#ifndef CORE_PARALLEL_DISTRIBUTED_DUALTREE_DFS_DEV_H
#define CORE_PARALLEL_DISTRIBUTED_DUALTREE_DFS_DEV_H

#include <boost/bind.hpp>
#include <boost/mpi.hpp>
#include <boost/tuple/tuple.hpp>
#include <map>
#include <queue>
#include "core/parallel/distributed_dualtree_dfs.h"
#include "core/parallel/distributed_dualtree_task_queue.h"
#include "core/gnp/dualtree_dfs_dev.h"
#include "core/parallel/message_tag.h"
#include "core/table/table.h"
#include "core/table/memory_mapped_file.h"

namespace core {
namespace parallel {

template<typename ProblemType>
int DistributedDualtreeDfs<ProblemType>::num_deterministic_prunes() const {
  return num_deterministic_prunes_;
}

template<typename ProblemType>
int DistributedDualtreeDfs<ProblemType>::num_probabilistic_prunes() const {
  return num_probabilistic_prunes_;
}

template<typename DistributedProblemType>
void DistributedDualtreeDfs <
DistributedProblemType >::HashSendList_(
  const std::pair<int, int> &local_rnode_id,
  int query_process_id,
  std::vector <
  core::parallel::RouteRequest<SubTableType> > *
  hashed_essential_reference_subtrees) {

  // May consider using a STL map to speed up the hashing.
  int found_index = -1;
  for(unsigned int i = 0;
      i < hashed_essential_reference_subtrees->size(); i++) {

    if((*hashed_essential_reference_subtrees)[i].object().has_same_subtable_id(
          local_rnode_id)) {
      found_index = i;
      break;
    }
  }
  if(found_index < 0) {
    hashed_essential_reference_subtrees->resize(
      hashed_essential_reference_subtrees->size() + 1);
    found_index = hashed_essential_reference_subtrees->size() - 1;
    (*hashed_essential_reference_subtrees)[ found_index ].Init(*world_);
    (*hashed_essential_reference_subtrees)[
      found_index].object().Init(
        reference_table_->local_table(),
        reference_table_->local_table()->get_tree()->FindByBeginCount(
          local_rnode_id.first, local_rnode_id.second), false);
    (*hashed_essential_reference_subtrees)[
      found_index].set_object_is_valid_flag(true);
  }
  (*hashed_essential_reference_subtrees)[
    found_index].add_destination(query_process_id);
}

template<typename DistributedProblemType>
template<typename MetricType>
void DistributedDualtreeDfs <
DistributedProblemType >::ComputeEssentialReferenceSubtrees_(
  const MetricType &metric_in,
  int max_reference_subtree_size,
  DistributedTreeType *global_query_node, TreeType *local_reference_node,
  std::vector <
  std::vector< std::pair<int, int> > > *essential_reference_subtrees,
  std::vector <
  core::parallel::RouteRequest<SubTableType> > *hashed_essential_reference_subtrees,
  std::vector< unsigned long int > *num_reference_points_assigned_per_process,
  std::vector <
  std::vector< core::math::Range > > *squared_distance_ranges,
  std::vector< unsigned long int > *extrinsic_prunes) {

  // Compute the squared distance ranges between the query node and
  // the reference node.
  core::math::Range squared_distance_range =
    global_query_node->bound().RangeDistanceSq(
      metric_in, local_reference_node->bound());

  // If the pair is prunable, then return.
  if((! weak_scaling_measuring_mode_) &&
      problem_->global().ConsiderExtrinsicPrune(squared_distance_range)) {
    typename TableType::TreeIterator qnode_it =
      query_table_->get_node_iterator(global_query_node);
    while(qnode_it.HasNext()) {
      int query_process_id;
      qnode_it.Next(&query_process_id);
      (*extrinsic_prunes)[query_process_id] += local_reference_node->count();
    }
    return;
  }

  if(global_query_node->is_leaf()) {

    // If the reference node size is within the size limit, then for
    // each query process in the set, add the reference node to the
    // list.
    if(local_reference_node->count() <= max_reference_subtree_size ||
        local_reference_node->is_leaf()) {
      typename TableType::TreeIterator qnode_it =
        query_table_->get_node_iterator(global_query_node);
      std::pair<int, int> local_rnode_id(
        local_reference_node->begin(), local_reference_node->count());
      while(qnode_it.HasNext()) {
        int query_process_id;
        qnode_it.Next(&query_process_id);

        // Only add if, weak scaling measuring is disabled or the
        // query process rank differs from the reference process rank
        // by at most 2 bits.
        if((! weak_scaling_measuring_mode_)  ||
            core::math::BitCount(query_process_id, world_->rank()) <= 2) {
          (*essential_reference_subtrees)[
            query_process_id].push_back(local_rnode_id);
          (* num_reference_points_assigned_per_process)[query_process_id ] +=
            local_rnode_id.second;

          // Add the query process ID to the list of query processes
          // that this reference subtree needs to be sent to.
          if(query_process_id != world_->rank()) {
            HashSendList_(
              local_rnode_id, query_process_id,
              hashed_essential_reference_subtrees);
          }

          // Push in the squared distance range.
          (*squared_distance_ranges)[query_process_id].push_back(
            squared_distance_range);
        }

        // Otherwise, consider it as pruned.
        else {
          (*extrinsic_prunes)[
            query_process_id] += local_reference_node->count();
        }<commit_msg>Bug fix in measuring weak scalability.<commit_after>/** @file distributed_dualtree_dfs_dev.h
 *
 *  The generic algorithm template for distributed dual-tree
 *  computation.
 *
 *  @author <NAME> (<EMAIL>)
 */

#ifndef CORE_PARALLEL_DISTRIBUTED_DUALTREE_DFS_DEV_H
#define CORE_PARALLEL_DISTRIBUTED_DUALTREE_DFS_DEV_H

#include <boost/bind.hpp>
#include <boost/mpi.hpp>
#include <boost/tuple/tuple.hpp>
#include <map>
#include <queue>
#include "core/parallel/distributed_dualtree_dfs.h"
#include "core/parallel/distributed_dualtree_task_queue.h"
#include "core/gnp/dualtree_dfs_dev.h"
#include "core/parallel/message_tag.h"
#include "core/table/table.h"
#include "core/table/memory_mapped_file.h"

namespace core {
namespace parallel {

template<typename ProblemType>
int DistributedDualtreeDfs<ProblemType>::num_deterministic_prunes() const {
  return num_deterministic_prunes_;
}

template<typename ProblemType>
int DistributedDualtreeDfs<ProblemType>::num_probabilistic_prunes() const {
  return num_probabilistic_prunes_;
}

template<typename DistributedProblemType>
void DistributedDualtreeDfs <
DistributedProblemType >::HashSendList_(
  const std::pair<int, int> &local_rnode_id,
  int query_process_id,
  std::vector <
  core::parallel::RouteRequest<SubTableType> > *
  hashed_essential_reference_subtrees) {

  // May consider using a STL map to speed up the hashing.
  int found_index = -1;
  for(unsigned int i = 0;
      i < hashed_essential_reference_subtrees->size(); i++) {

    if((*hashed_essential_reference_subtrees)[i].object().has_same_subtable_id(
          local_rnode_id)) {
      found_index = i;
      break;
    }
  }
  if(found_index < 0) {
    hashed_essential_reference_subtrees->resize(
      hashed_essential_reference_subtrees->size() + 1);
    found_index = hashed_essential_reference_subtrees->size() - 1;
    (*hashed_essential_reference_subtrees)[ found_index ].Init(*world_);
    (*hashed_essential_reference_subtrees)[
      found_index].object().Init(
        reference_table_->local_table(),
        reference_table_->local_table()->get_tree()->FindByBeginCount(
          local_rnode_id.first, local_rnode_id.second), false);
    (*hashed_essential_reference_subtrees)[
      found_index].set_object_is_valid_flag(true);
  }
  (*hashed_essential_reference_subtrees)[
    found_index].add_destination(query_process_id);
}

template<typename DistributedProblemType>
template<typename MetricType>
void DistributedDualtreeDfs <
DistributedProblemType >::ComputeEssentialReferenceSubtrees_(
  const MetricType &metric_in,
  int max_reference_subtree_size,
  DistributedTreeType *global_query_node, TreeType *local_reference_node,
  std::vector <
  std::vector< std::pair<int, int> > > *essential_reference_subtrees,
  std::vector <
  core::parallel::RouteRequest<SubTableType> > *hashed_essential_reference_subtrees,
  std::vector< unsigned long int > *num_reference_points_assigned_per_process,
  std::vector <
  std::vector< core::math::Range > > *squared_distance_ranges,
  std::vector< unsigned long int > *extrinsic_prunes) {

  // Compute the squared distance ranges between the query node and
  // the reference node.
  core::math::Range squared_distance_range =
    global_query_node->bound().RangeDistanceSq(
      metric_in, local_reference_node->bound());

  // If the pair is prunable, then return.
  if((! weak_scaling_measuring_mode_) &&
      problem_->global().ConsiderExtrinsicPrune(squared_distance_range)) {
    typename TableType::TreeIterator qnode_it =
      query_table_->get_node_iterator(global_query_node);
    while(qnode_it.HasNext()) {
      int query_process_id;
      qnode_it.Next(&query_process_id);
      (*extrinsic_prunes)[query_process_id] += local_reference_node->count();
    }
    return;
  }

  if(global_query_node->is_leaf()) {

    // If the reference node size is within the size limit, then for
    // each query process in the set, add the reference node to the
    // list.
    if(local_reference_node->count() <= max_reference_subtree_size ||
        local_reference_node->is_leaf()) {
      typename TableType::TreeIterator qnode_it =
        query_table_->get_node_iterator(global_query_node);
      std::pair<int, int> local_rnode_id(
        local_reference_node->begin(), local_reference_node->count());
      while(qnode_it.HasNext()) {
        int query_process_id;
        qnode_it.Next(&query_process_id);

        // Only add if, weak scaling measuring is disabled or the
        // query process rank differs from the reference process rank
        // by at most 2 bits.
        if( (! weak_scaling_measuring_mode_)  ||
	   ( core::math::BitCount(query_process_id, world_->rank()) <= 2 &&
	     (* num_reference_points_assigned_per_process)[query_process_id ] <
	     max_num_reference_points_to_pack_per_process_ ) ) {
          (*essential_reference_subtrees)[
            query_process_id].push_back(local_rnode_id);
          (* num_reference_points_assigned_per_process)[query_process_id ] +=
            local_rnode_id.second;

          // Add the query process ID to the list of query processes
          // that this reference subtree needs to be sent to.
          if(query_process_id != world_->rank()) {
            HashSendList_(
              local_rnode_id, query_process_id,
              hashed_essential_reference_subtrees);
          }

          // Push in the squared distance range.
          (*squared_distance_ranges)[query_process_id].push_back(
            squared_distance_range);
        }

        // Otherwise, consider it as pruned.
        else {
          (*extrinsic_prunes)[
            query_process_id] += local_reference_node->count();
        }
{'fraction_non_alphanumeric': 0.10288212724190238, 'fraction_numerical': 0.0012492192379762648, 'mean_word_length': 4.10150204824761, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4337332', 'n_tokens_mistral': 3631, 'n_tokens_neox': 3461, 'n_words': 641}