<commit_before>/*
   Copyright 2014 Software Reliability Lab, ETH Zurich

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 */

#ifndef N2_INFERENCE_GRAPH_INFERENCE_H_
#define N2_INFERENCE_GRAPH_INFERENCE_H_

#include <unordered_map>
#include <google/dense_hash_map>
#include <string.h>
#include <iterator>

#include "inference.h"
#include "lock_free_weight.h"
#include "stringset.h"
#include "maputil.h"
#include "label_checker.h"


typedef std::multiset<int> Factor;


namespace std {
  template <> struct hash<std::vector<int>> {
    size_t operator()(const std::vector<int>& x) const {
      int hc = x.size();
      for (unsigned int i = 0; i < x.size(); i++) {
        hc = hc * 6037 + x[i];
      }
      return hc;
    }
  };
}

namespace std {
  template <> struct hash<std::multiset<int>> {
    size_t operator()(const std::multiset<int>& x) const {
      int hc = x.size();
      for (auto var = x.begin(); var != x.end(); var++) {
        hc = hc * 6037 + *var;
      }
      return hc;
    }
  };
}

struct NodeConfusionStats {
  NodeConfusionStats() : num_non_confusable_nodes(0), num_confusable_nodes(0), num_expected_confusions(0) { }

  int num_non_confusable_nodes;
  int num_confusable_nodes;
  int num_expected_confusions;
};

struct FactorFeaturesLevel {
  FactorFeaturesLevel() : factor_features(std::vector<std::pair<double, Factor>>()), next_level(std::unordered_map<int, std::shared_ptr<FactorFeaturesLevel>>()) {}

  ~FactorFeaturesLevel() {
    factor_features.clear();
    next_level.clear();
  }

  void InsertFactorFeature(double weight, Factor& f, int current_depth, int maximum_depth, int current_label, Factor visited_labels) {
    factor_features.push_back(std::pair<double, Factor>(weight, f));
    Factor next_level_labels_visited;
    if (current_label > 0) {
      visited_labels.insert(current_label);
    }
    if (current_depth < maximum_depth && visited_labels.size() < f.size()) {
      for (auto it = f.begin(); it != f.end(); ++it) {
        if ((visited_labels.count(*it) + next_level_labels_visited.count(*it)) < f.count(*it)) {
          next_level_labels_visited.insert(*it);
          if (next_level.count(*it) == 0) {
            next_level[*it] = std::make_shared<FactorFeaturesLevel>();
          }
          next_level[*it]->InsertFactorFeature(weight, f, current_depth + 1, maximum_depth, *it, visited_labels);
        }
      }
    }
  }

  void GetFactors(Factor& giv_labels, int current_depth, int next_level_label, std::vector<Factor>* candidates, size_t beam_size) {
    if (factor_features.size() < beam_size || next_level.empty()) {
      for (auto it = factor_features.begin(); it != factor_features.end(); ++it) {
        candidates->push_back(it->second);
      }
    } else {
      auto it = giv_labels.begin();
      std::advance(it, current_depth - 1);
      if (next_level.count(next_level_label) > 0) {
        next_level[next_level_label]->GetFactors(giv_labels, current_depth + 1, *it, candidates, beam_size);
      }
    }
  }<commit_msg>Fix segmentation fault in GetFactors function<commit_after>/*
   Copyright 2014 Software Reliability Lab, ETH Zurich

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 */

#ifndef N2_INFERENCE_GRAPH_INFERENCE_H_
#define N2_INFERENCE_GRAPH_INFERENCE_H_

#include <unordered_map>
#include <google/dense_hash_map>
#include <string.h>
#include <iterator>

#include "inference.h"
#include "lock_free_weight.h"
#include "stringset.h"
#include "maputil.h"
#include "label_checker.h"


typedef std::multiset<int> Factor;


namespace std {
  template <> struct hash<std::vector<int>> {
    size_t operator()(const std::vector<int>& x) const {
      int hc = x.size();
      for (unsigned int i = 0; i < x.size(); i++) {
        hc = hc * 6037 + x[i];
      }
      return hc;
    }
  };
}

namespace std {
  template <> struct hash<std::multiset<int>> {
    size_t operator()(const std::multiset<int>& x) const {
      int hc = x.size();
      for (auto var = x.begin(); var != x.end(); var++) {
        hc = hc * 6037 + *var;
      }
      return hc;
    }
  };
}

struct NodeConfusionStats {
  NodeConfusionStats() : num_non_confusable_nodes(0), num_confusable_nodes(0), num_expected_confusions(0) { }

  int num_non_confusable_nodes;
  int num_confusable_nodes;
  int num_expected_confusions;
};

struct FactorFeaturesLevel {
  FactorFeaturesLevel() : factor_features(std::vector<std::pair<double, Factor>>()), next_level(std::unordered_map<int, std::shared_ptr<FactorFeaturesLevel>>()) {}

  ~FactorFeaturesLevel() {
    factor_features.clear();
    next_level.clear();
  }

  void InsertFactorFeature(double weight, Factor& f, int current_depth, int maximum_depth, int current_label, Factor visited_labels) {
    factor_features.push_back(std::pair<double, Factor>(weight, f));
    Factor next_level_labels_visited;
    if (current_label > 0) {
      visited_labels.insert(current_label);
    }
    if (current_depth < maximum_depth && visited_labels.size() < f.size()) {
      for (auto it = f.begin(); it != f.end(); ++it) {
        if ((visited_labels.count(*it) + next_level_labels_visited.count(*it)) < f.count(*it)) {
          next_level_labels_visited.insert(*it);
          if (next_level.count(*it) == 0) {
            next_level[*it] = std::make_shared<FactorFeaturesLevel>();
          }
          next_level[*it]->InsertFactorFeature(weight, f, current_depth + 1, maximum_depth, *it, visited_labels);
        }
      }
    }
  }

  void GetFactors(Factor& giv_labels, int current_depth, int next_level_label, std::vector<Factor>* candidates, size_t beam_size) {
    if (factor_features.size() < beam_size || next_level.empty() || giv_labels.empty()) {
      for (auto it = factor_features.begin(); it != factor_features.end(); ++it) {
        candidates->push_back(it->second);
      }
    } else {
      auto it = giv_labels.begin();
      std::advance(it, current_depth);
      if (next_level.count(next_level_label) > 0) {
        next_level[next_level_label]->GetFactors(giv_labels, current_depth + 1, *it, candidates, beam_size);
      }
    }
  }
{'fraction_non_alphanumeric': 0.12747314867156587, 'fraction_numerical': 0.007772752967778406, 'mean_word_length': 3.692970822281167, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6792287', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2263, 'n_words': 557}