<commit_before>// Copyright (c) 2012 Cloudera, Inc. All rights reserved.

#include "util/runtime-profile.h"

#include "common/object-pool.h"
#include "util/debug-util.h"
#include "util/cpu-info.h"

#include <iostream>
#include <boost/thread/locks.hpp>

using namespace boost;
using namespace std;

namespace impala {

RuntimeProfile::RuntimeProfile(ObjectPool* pool, const string& name) :
  pool_(pool),
  name_(name),
  metadata_(-1),
  counter_total_time_(TCounterType::CPU_TICKS) {
  counter_map_["TotalTime"] = &counter_total_time_;
}

RuntimeProfile* RuntimeProfile::CreateFromThrift(ObjectPool* pool,
    const TRuntimeProfileTree& profiles) {
  if (profiles.nodes.size() == 0) return NULL;
  int idx = 0;
  return RuntimeProfile::CreateFromThrift(pool, profiles.nodes, &idx);
}

RuntimeProfile* RuntimeProfile::CreateFromThrift(ObjectPool* pool,
    const vector<TRuntimeProfileNode>& nodes, int* idx) {
  DCHECK_LT(*idx, nodes.size());

  const TRuntimeProfileNode& node = nodes[*idx];
  RuntimeProfile* profile = pool->Add(new RuntimeProfile(pool, node.name));
  profile->metadata_ = node.metadata;
  for (int i = 0; i < node.counters.size(); ++i) {
    const TCounter& counter = node.counters[i];
    profile->counter_map_[counter.name] =
      pool->Add(new Counter(counter.type, counter.value));
  }
  ++*idx;
  for (int i = 0; i < node.num_children; ++i) {
    profile->AddChild(RuntimeProfile::CreateFromThrift(pool, nodes, idx));
  }
  return profile;
}

void RuntimeProfile::Merge(RuntimeProfile* other) {
  // Merge this level
  map<string, Counter*>::iterator dst_iter;
  map<string, Counter*>::const_iterator src_iter;
  {
    lock_guard<mutex> l(counter_map_lock_);
    lock_guard<mutex> m(other->counter_map_lock_);
    for (src_iter = other->counter_map_.begin();
         src_iter != other->counter_map_.end(); ++src_iter) {
      dst_iter = counter_map_.find(src_iter->first);
      if (dst_iter == counter_map_.end()) {
        counter_map_[src_iter->first] =
          pool_->Add(new Counter(src_iter->second->type(), src_iter->second->value()));
      } else {
        if (dst_iter->second->type() != src_iter->second->type()) {
          LOG(ERROR) << "Cannot merge counters with the same name (" <<
           dst_iter->first << ") but different types.";
        } else {
          dst_iter->second->Update(src_iter->second->value());
        }
      }
    }
  }

  {
    lock_guard<mutex> l(children_lock_);
    lock_guard<mutex> m(other->children_lock_);
    // Recursively merge children with matching names
    for (int i = 0; i < other->children_.size(); ++i) {
      RuntimeProfile* other_child = other->children_[i].first;
      ChildMap::iterator j = child_map_.find(other_child->name_);
      RuntimeProfile* child = NULL;
      if (j != child_map_.end()) {
        child = j->second;
      } else {
        child = pool_->Add(new RuntimeProfile(pool_, other_child->name_));
        bool indent_other_child = other->children_[i].second;
        child_map_[child->name_] = child;
        children_.push_back(make_pair(child, indent_other_child));
      }
      child->Merge(other_child);
    }
  }
}

void RuntimeProfile::Update(const TRuntimeProfileTree& thrift_profile) {
  int idx = 0;
  Update(thrift_profile.nodes, &idx);
  DCHECK_EQ(idx, thrift_profile.nodes.size());
}

void RuntimeProfile::Update(const vector<TRuntimeProfileNode>& nodes, int* idx) {
  DCHECK_LT(*idx, nodes.size());
  const TRuntimeProfileNode& node = nodes[*idx];
  {
    lock_guard<mutex> l(counter_map_lock_);
    // update this level
    map<string, Counter*>::iterator dst_iter;
    for (int i = 0; i < node.counters.size(); ++i) {
      const TCounter& tcounter = node.counters[i];
      CounterMap::iterator j = counter_map_.find(tcounter.name);
      if (j == counter_map_.end()) {
        counter_map_[tcounter.name] =
          pool_->Add(new Counter(tcounter.type, tcounter.value));
      } else {
        if (j->second->type() != tcounter.type) {
          LOG(ERROR) << "Cannot update counters with the same name (" <<
              j->first << ") but different types.";
        } else {
          j->second->Set(tcounter.value);
        }
      }
    }
  }

  ++*idx;
  {
    lock_guard<mutex> l(children_lock_);
    // update children with matching names; create new ones if they don't match
    for (int i = 0; i < node.num_children; ++i) {
      const TRuntimeProfileNode& tchild = nodes[*idx];
      ChildMap::iterator j = child_map_.find(tchild.name);
      RuntimeProfile* child = NULL;
      if (j != child_map_.end()) {
        child = j->second;
      } else {
        child = pool_->Add(new RuntimeProfile(pool_, tchild.name));
        child_map_[tchild.name] = child;
        children_.push_back(make_pair(child, tchild.indent));<commit_msg>Fix metadata for runtime profiles.<commit_after>// Copyright (c) 2012 Cloudera, Inc. All rights reserved.

#include "util/runtime-profile.h"

#include "common/object-pool.h"
#include "util/debug-util.h"
#include "util/cpu-info.h"

#include <iostream>
#include <boost/thread/locks.hpp>

using namespace boost;
using namespace std;

namespace impala {

RuntimeProfile::RuntimeProfile(ObjectPool* pool, const string& name) :
  pool_(pool),
  name_(name),
  metadata_(-1),
  counter_total_time_(TCounterType::CPU_TICKS) {
  counter_map_["TotalTime"] = &counter_total_time_;
}

RuntimeProfile* RuntimeProfile::CreateFromThrift(ObjectPool* pool,
    const TRuntimeProfileTree& profiles) {
  if (profiles.nodes.size() == 0) return NULL;
  int idx = 0;
  return RuntimeProfile::CreateFromThrift(pool, profiles.nodes, &idx);
}

RuntimeProfile* RuntimeProfile::CreateFromThrift(ObjectPool* pool,
    const vector<TRuntimeProfileNode>& nodes, int* idx) {
  DCHECK_LT(*idx, nodes.size());

  const TRuntimeProfileNode& node = nodes[*idx];
  RuntimeProfile* profile = pool->Add(new RuntimeProfile(pool, node.name));
  profile->metadata_ = node.metadata;
  for (int i = 0; i < node.counters.size(); ++i) {
    const TCounter& counter = node.counters[i];
    profile->counter_map_[counter.name] =
      pool->Add(new Counter(counter.type, counter.value));
  }
  ++*idx;
  for (int i = 0; i < node.num_children; ++i) {
    profile->AddChild(RuntimeProfile::CreateFromThrift(pool, nodes, idx));
  }
  return profile;
}

void RuntimeProfile::Merge(RuntimeProfile* other) {
  // Merge this level
  map<string, Counter*>::iterator dst_iter;
  map<string, Counter*>::const_iterator src_iter;
  {
    lock_guard<mutex> l(counter_map_lock_);
    lock_guard<mutex> m(other->counter_map_lock_);
    for (src_iter = other->counter_map_.begin();
         src_iter != other->counter_map_.end(); ++src_iter) {
      dst_iter = counter_map_.find(src_iter->first);
      if (dst_iter == counter_map_.end()) {
        counter_map_[src_iter->first] =
          pool_->Add(new Counter(src_iter->second->type(), src_iter->second->value()));
      } else {
        if (dst_iter->second->type() != src_iter->second->type()) {
          LOG(ERROR) << "Cannot merge counters with the same name (" <<
           dst_iter->first << ") but different types.";
        } else {
          dst_iter->second->Update(src_iter->second->value());
        }
      }
    }
  }

  {
    lock_guard<mutex> l(children_lock_);
    lock_guard<mutex> m(other->children_lock_);
    // Recursively merge children with matching names
    for (int i = 0; i < other->children_.size(); ++i) {
      RuntimeProfile* other_child = other->children_[i].first;
      ChildMap::iterator j = child_map_.find(other_child->name_);
      RuntimeProfile* child = NULL;
      if (j != child_map_.end()) {
        child = j->second;
      } else {
        child = pool_->Add(new RuntimeProfile(pool_, other_child->name_));
        child->metadata_ = other_child->metadata_;
        bool indent_other_child = other->children_[i].second;
        child_map_[child->name_] = child;
        children_.push_back(make_pair(child, indent_other_child));
      }
      child->Merge(other_child);
    }
  }
}

void RuntimeProfile::Update(const TRuntimeProfileTree& thrift_profile) {
  int idx = 0;
  Update(thrift_profile.nodes, &idx);
  DCHECK_EQ(idx, thrift_profile.nodes.size());
}

void RuntimeProfile::Update(const vector<TRuntimeProfileNode>& nodes, int* idx) {
  DCHECK_LT(*idx, nodes.size());
  const TRuntimeProfileNode& node = nodes[*idx];
  {
    lock_guard<mutex> l(counter_map_lock_);
    // update this level
    map<string, Counter*>::iterator dst_iter;
    for (int i = 0; i < node.counters.size(); ++i) {
      const TCounter& tcounter = node.counters[i];
      CounterMap::iterator j = counter_map_.find(tcounter.name);
      if (j == counter_map_.end()) {
        counter_map_[tcounter.name] =
          pool_->Add(new Counter(tcounter.type, tcounter.value));
      } else {
        if (j->second->type() != tcounter.type) {
          LOG(ERROR) << "Cannot update counters with the same name (" <<
              j->first << ") but different types.";
        } else {
          j->second->Set(tcounter.value);
        }
      }
    }
  }

  ++*idx;
  {
    lock_guard<mutex> l(children_lock_);
    // update children with matching names; create new ones if they don't match
    for (int i = 0; i < node.num_children; ++i) {
      const TRuntimeProfileNode& tchild = nodes[*idx];
      ChildMap::iterator j = child_map_.find(tchild.name);
      RuntimeProfile* child = NULL;
      if (j != child_map_.end()) {
        child = j->second;
      } else {
        child = pool_->Add(new RuntimeProfile(pool_, tchild.name));
        child->metadata_ = tchild.metadata;
        child_map_[tchild.name] = child;
        children_.push_back(make_pair(child, tchild.indent));
{'fraction_non_alphanumeric': 0.14657845772172834, 'fraction_numerical': 0.0026876162910895183, 'mean_word_length': 3.4178082191780823, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '901555', 'n_tokens_mistral': 3116, 'n_tokens_neox': 3061, 'n_words': 643}