<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/metrics/thread_watcher.h"

#include <math.h>  // ceil

#include "base/bind.h"
#include "base/compiler_specific.h"
#include "base/debug/alias.h"
#include "base/debug/debugger.h"
#include "base/debug/dump_without_crashing.h"
#include "base/lazy_instance.h"
#include "base/metrics/field_trial.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_split.h"
#include "base/strings/string_tokenizer.h"
#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
#include "build/build_config.h"
#include "chrome/browser/chrome_notification_types.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/common/chrome_version_info.h"
#include "chrome/common/logging_chrome.h"
#include "content/public/browser/notification_service.h"

#if defined(OS_WIN)
#include "base/win/windows_version.h"
#endif

using content::BrowserThread;

namespace {

// The following are unique function names for forcing the crash when a thread
// is unresponsive. This makes it possible to tell from the callstack alone what
// thread was unresponsive.
//
// We disable optimizations for this block of functions so the compiler doesn't
// merge them all together.
MSVC_DISABLE_OPTIMIZE()
MSVC_PUSH_DISABLE_WARNING(4748)

void ReportThreadHang() {
#if defined(NDEBUG)
  base::debug::DumpWithoutCrashing();
#else
  base::debug::BreakDebugger();
#endif
}

#if !defined(OS_ANDROID) || !defined(NDEBUG)
// TODO(rtenneti): Enabled crashing, after getting data.
NOINLINE void StartupHang() {
  ReportThreadHang();
}
#endif  // OS_ANDROID

NOINLINE void ShutdownHang() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_UI() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_DB() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_FILE() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_FILE_USER_BLOCKING() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_PROCESS_LAUNCHER() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_CACHE() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_IO() {
  ReportThreadHang();
}

void CrashBecauseThreadWasUnresponsive(BrowserThread::ID thread_id) {
  base::debug::Alias(&thread_id);

  switch (thread_id) {
    case BrowserThread::UI:
      return ThreadUnresponsive_UI();
    case BrowserThread::DB:
      return ThreadUnresponsive_DB();
    case BrowserThread::FILE:
      return ThreadUnresponsive_FILE();
    case BrowserThread::FILE_USER_BLOCKING:
      return ThreadUnresponsive_FILE_USER_BLOCKING();
    case BrowserThread::PROCESS_LAUNCHER:
      return ThreadUnresponsive_PROCESS_LAUNCHER();
    case BrowserThread::CACHE:
      return ThreadUnresponsive_CACHE();
    case BrowserThread::IO:
      return ThreadUnresponsive_IO();
    case BrowserThread::ID_COUNT:
      CHECK(false);  // This shouldn't actually be reached!
      break;

    // Omission of the default hander is intentional -- that way the compiler
    // should warn if our switch becomes outdated.
  }

  CHECK(false) << "Unknown thread was unresponsive.";  // Shouldn't be reached.
}

MSVC_POP_WARNING()
MSVC_ENABLE_OPTIMIZE();

}  // namespace

// ThreadWatcher methods and members.
ThreadWatcher::ThreadWatcher(const WatchingParams& params)
    : thread_id_(params.thread_id),
      thread_name_(params.thread_name),
      watched_loop_(
          BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
      sleep_time_(params.sleep_time),
      unresponsive_time_(params.unresponsive_time),
      ping_time_(base::TimeTicks::Now()),
      pong_time_(ping_time_),
      ping_sequence_number_(0),
      active_(false),
      ping_count_(params.unresponsive_threshold),
      response_time_histogram_(NULL),
      unresponsive_time_histogram_(NULL),
      unresponsive_count_(0),
      hung_processing_complete_(false),
      unresponsive_threshold_(params.unresponsive_threshold),
      crash_on_hang_(params.crash_on_hang),
      live_threads_threshold_(params.live_threads_threshold),
      weak_ptr_factory_(this) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  Initialize();
}

ThreadWatcher::~ThreadWatcher() {}

// static
void ThreadWatcher::StartWatching(const WatchingParams& params) {
  DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  DCHECK_GE(params.unresponsive_time.InMilliseconds(),
            params.sleep_time.InMilliseconds());

  // If we are not on WatchDogThread, then post a task to call StartWatching on
  // WatchDogThread.
  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
    WatchDogThread::PostTask(
        FROM_HERE,
        base::Bind(&ThreadWatcher::StartWatching, params));
    return;
  }

  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  // Create a new thread watcher object for the given thread and activate it.
  ThreadWatcher* watcher = new ThreadWatcher(params);

  DCHECK(watcher);
  // If we couldn't register the thread watcher object, we are shutting down,
  // then don't activate thread watching.
  if (!ThreadWatcherList::IsRegistered(params.thread_id))
    return;
  watcher->ActivateThreadWatching();
}

void ThreadWatcher::ActivateThreadWatching() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  if (active_) return;
  active_ = true;
  ping_count_ = unresponsive_threshold_;
  ResetHangCounters();
  base::MessageLoop::current()->PostTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::PostPingMessage,
                 weak_ptr_factory_.GetWeakPtr()));
}

void ThreadWatcher::DeActivateThreadWatching() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  active_ = false;
  ping_count_ = 0;
  weak_ptr_factory_.InvalidateWeakPtrs();
}

void ThreadWatcher::WakeUp() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // There is some user activity, PostPingMessage task of thread watcher if
  // needed.
  if (!active_) return;

  // Throw away the previous |unresponsive_count_| and start over again. Just
  // before going to sleep, |unresponsive_count_| could be very close to
  // |unresponsive_threshold_| and when user becomes active,
  // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
  // response for ping messages. Reset |unresponsive_count_| to start measuring
  // the unresponsiveness of the threads when system becomes active.
  unresponsive_count_ = 0;

  if (ping_count_ <= 0) {
    ping_count_ = unresponsive_threshold_;
    ResetHangCounters();
    PostPingMessage();
  } else {
    ping_count_ = unresponsive_threshold_;
  }
}

void ThreadWatcher::PostPingMessage() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // If we have stopped watching or if the user is idle, then stop sending
  // ping messages.
  if (!active_ || ping_count_ <= 0)
    return;

  // Save the current time when we have sent ping message.
  ping_time_ = base::TimeTicks::Now();

  // Send a ping message to the watched thread. Callback will be called on
  // the WatchDogThread.
  base::Closure callback(
      base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
                 ping_sequence_number_));
  if (watched_loop_->PostTask(
          FROM_HERE,
          base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
                     callback))) {
      // Post a task to check the responsiveness of watched thread.
      base::MessageLoop::current()->PostDelayedTask(
          FROM_HERE,
          base::Bind(&ThreadWatcher::OnCheckResponsiveness,
                     weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
          unresponsive_time_);
  } else {
    // Watched thread might have gone away, stop watching it.
    DeActivateThreadWatching();
  }
}

void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  // Record watched thread's response time.
  base::TimeTicks now = base::TimeTicks::Now();
  base::TimeDelta response_time = now - ping_time_;
  response_time_histogram_->AddTime(response_time);

  // Save the current time when we have got pong message.
  pong_time_ = now;

  // Check if there are any extra pings in flight.
  DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
  if (ping_sequence_number_ != ping_sequence_number)
    return;

  // Increment sequence number for the next ping message to indicate watched
  // thread is responsive.
  ++ping_sequence_number_;

  // If we have stopped watching or if the user is idle, then stop sending
  // ping messages.
  if (!active_ || --ping_count_ <= 0)
    return;

  base::MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::PostPingMessage,
                 weak_ptr_factory_.GetWeakPtr()),
      sleep_time_);
}

void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // If we have stopped watching then consider thread as responding.
  if (!active_) {
    responsive_ = true;
    return;
  }
  // If the latest ping_sequence_number_ is not same as the ping_sequence_number
  // that is passed in, then we can assume OnPongMessage was called.
  // OnPongMessage increments ping_sequence_number_.
  if (ping_sequence_number_ != ping_sequence_number) {
    // Reset unresponsive_count_ to zero because we got a response from the
    // watched thread.
    ResetHangCounters();

    responsive_ = true;
    return;
  }
  // Record that we got no response from watched thread.
  GotNoResponse();

  // Post a task to check the responsiveness of watched thread.
  base::MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::OnCheckResponsiveness,
                 weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
      unresponsive_time_);
  responsive_ = false;
}

void ThreadWatcher::Initialize() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  ThreadWatcherList::Register(this);

  const std::string response_time_histogram_name =
      "ThreadWatcher.ResponseTime." + thread_name_;
  response_time_histogram_ = base::Histogram::FactoryTimeGet(
      response_time_histogram_name,
      base::TimeDelta::FromMilliseconds(1),
      base::TimeDelta::FromSeconds(100), 50,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string unresponsive_time_histogram_name =
      "ThreadWatcher.Unresponsive." + thread_name_;
  unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
      unresponsive_time_histogram_name,
      base::TimeDelta::FromMilliseconds(1),
      base::TimeDelta::FromSeconds(100), 50,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string responsive_count_histogram_name =
      "ThreadWatcher.ResponsiveThreads." + thread_name_;
  responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
      responsive_count_histogram_name, 1, 10, 11,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string unresponsive_count_histogram_name =
      "ThreadWatcher.UnresponsiveThreads." + thread_name_;
  unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
      unresponsive_count_histogram_name, 1, 10, 11,
      base::Histogram::kUmaTargetedHistogramFlag);
}

// static
void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
                                  const base::Closure& callback_task) {
  // This method is called on watched thread.
  DCHECK(BrowserThread::CurrentlyOn(thread_id));
  WatchDogThread::PostTask(FROM_HERE, callback_task);
}

void ThreadWatcher::ResetHangCounters() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  unresponsive_count_ = 0;
  hung_processing_complete_ = false;
}

void ThreadWatcher::GotNoResponse() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  ++unresponsive_count_;
  if (!IsVeryUnresponsive())
    return;

  // Record total unresponsive_time since last pong message.
  base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
  unresponsive_time_histogram_->AddTime(unresponse_time);

  // We have already collected stats for the non-responding watched thread.
  if (hung_processing_complete_)
    return;

  // Record how other threads are responding.
  uint32 responding_thread_count = 0;
  uint32 unresponding_thread_count = 0;
  ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
                                        &unresponding_thread_count);

  // Record how many watched threads are responding.
  responsive_count_histogram_->Add(responding_thread_count);

  // Record how many watched threads are not responding.
  unresponsive_count_histogram_->Add(unresponding_thread_count);

  // Crash the browser if the watched thread is to be crashed on hang and if the
  // number of other threads responding is less than or equal to
  // live_threads_threshold_ and at least one other thread is responding.
  if (crash_on_hang_ &&
      responding_thread_count > 0 &&
      responding_thread_count <= live_threads_threshold_) {
    static bool crashed_once = false;
    if (!crashed_once) {
      crashed_once = true;
      CrashBecauseThreadWasUnresponsive(thread_id_);
    }
  }

  hung_processing_complete_ = true;
}

bool ThreadWatcher::IsVeryUnresponsive() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  return unresponsive_count_ >= unresponsive_threshold_;
}

// ThreadWatcherList methods and members.
//
// static
ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
// static
bool ThreadWatcherList::g_stopped_ = false;
// static
const int ThreadWatcherList::kSleepSeconds = 1;
// static
const int ThreadWatcherList::kUnresponsiveSeconds = 2;
// static
const int ThreadWatcherList::kUnresponsiveCount = 9;
// static
const int ThreadWatcherList::kLiveThreadsThreshold = 2;
// static, non-const for tests.
int ThreadWatcherList::g_initialize_delay_seconds = 120;

ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
    uint32 live_threads_threshold,
    uint32 unresponsive_threshold)
    : live_threads_threshold(live_threads_threshold),
      unresponsive_threshold(unresponsive_threshold) {
}

ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
    : live_threads_threshold(kLiveThreadsThreshold),
      unresponsive_threshold(kUnresponsiveCount) {
}

// static
void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
  // TODO(rtenneti): Enable ThreadWatcher.
  uint32 unresponsive_threshold;
  CrashOnHangThreadMap crash_on_hang_threads;
  ParseCommandLine(command_line,
                   &unresponsive_threshold,
                   &crash_on_hang_threads);

  ThreadWatcherObserver::SetupNotifications(
      base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));

  WatchDogThread::PostTask(
      FROM_HERE,
      base::Bind(&ThreadWatcherList::SetStopped, false));

  WatchDogThread::PostDelayedTask(
      FROM_HERE,
      base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
                 unresponsive_threshold,
                 crash_on_hang_threads),
      base::TimeDelta::FromSeconds(g_initialize_delay_seconds));
}

// static
void ThreadWatcherList::StopWatchingAll() {
  // TODO(rtenneti): Enable ThreadWatcher.
  ThreadWatcherObserver::RemoveNotifications();
  DeleteAll();
}

// static
void ThreadWatcherList::Register(ThreadWatcher* watcher) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  if (!g_thread_watcher_list_)
    return;
  DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
  g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
}

// static
bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {<commit_msg>ThreadWatcher - Disarm the StartupTimeBomb, if we haven't started ThreadWacther (for example, WatchDog thread is not started in Debug mode - CL: https://codereview.chromium.org/573683003/).<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/metrics/thread_watcher.h"

#include <math.h>  // ceil

#include "base/bind.h"
#include "base/compiler_specific.h"
#include "base/debug/alias.h"
#include "base/debug/debugger.h"
#include "base/debug/dump_without_crashing.h"
#include "base/lazy_instance.h"
#include "base/metrics/field_trial.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_split.h"
#include "base/strings/string_tokenizer.h"
#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
#include "build/build_config.h"
#include "chrome/browser/chrome_notification_types.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/common/chrome_version_info.h"
#include "chrome/common/logging_chrome.h"
#include "content/public/browser/notification_service.h"

#if defined(OS_WIN)
#include "base/win/windows_version.h"
#endif

using content::BrowserThread;

namespace {

// The following are unique function names for forcing the crash when a thread
// is unresponsive. This makes it possible to tell from the callstack alone what
// thread was unresponsive.
//
// We disable optimizations for this block of functions so the compiler doesn't
// merge them all together.
MSVC_DISABLE_OPTIMIZE()
MSVC_PUSH_DISABLE_WARNING(4748)

void ReportThreadHang() {
#if defined(NDEBUG)
  base::debug::DumpWithoutCrashing();
#else
  base::debug::BreakDebugger();
#endif
}

#if !defined(OS_ANDROID) || !defined(NDEBUG)
// TODO(rtenneti): Enabled crashing, after getting data.
NOINLINE void StartupHang() {
  ReportThreadHang();
}
#endif  // OS_ANDROID

NOINLINE void ShutdownHang() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_UI() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_DB() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_FILE() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_FILE_USER_BLOCKING() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_PROCESS_LAUNCHER() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_CACHE() {
  ReportThreadHang();
}

NOINLINE void ThreadUnresponsive_IO() {
  ReportThreadHang();
}

void CrashBecauseThreadWasUnresponsive(BrowserThread::ID thread_id) {
  base::debug::Alias(&thread_id);

  switch (thread_id) {
    case BrowserThread::UI:
      return ThreadUnresponsive_UI();
    case BrowserThread::DB:
      return ThreadUnresponsive_DB();
    case BrowserThread::FILE:
      return ThreadUnresponsive_FILE();
    case BrowserThread::FILE_USER_BLOCKING:
      return ThreadUnresponsive_FILE_USER_BLOCKING();
    case BrowserThread::PROCESS_LAUNCHER:
      return ThreadUnresponsive_PROCESS_LAUNCHER();
    case BrowserThread::CACHE:
      return ThreadUnresponsive_CACHE();
    case BrowserThread::IO:
      return ThreadUnresponsive_IO();
    case BrowserThread::ID_COUNT:
      CHECK(false);  // This shouldn't actually be reached!
      break;

    // Omission of the default hander is intentional -- that way the compiler
    // should warn if our switch becomes outdated.
  }

  CHECK(false) << "Unknown thread was unresponsive.";  // Shouldn't be reached.
}

MSVC_POP_WARNING()
MSVC_ENABLE_OPTIMIZE();

}  // namespace

// ThreadWatcher methods and members.
ThreadWatcher::ThreadWatcher(const WatchingParams& params)
    : thread_id_(params.thread_id),
      thread_name_(params.thread_name),
      watched_loop_(
          BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
      sleep_time_(params.sleep_time),
      unresponsive_time_(params.unresponsive_time),
      ping_time_(base::TimeTicks::Now()),
      pong_time_(ping_time_),
      ping_sequence_number_(0),
      active_(false),
      ping_count_(params.unresponsive_threshold),
      response_time_histogram_(NULL),
      unresponsive_time_histogram_(NULL),
      unresponsive_count_(0),
      hung_processing_complete_(false),
      unresponsive_threshold_(params.unresponsive_threshold),
      crash_on_hang_(params.crash_on_hang),
      live_threads_threshold_(params.live_threads_threshold),
      weak_ptr_factory_(this) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  Initialize();
}

ThreadWatcher::~ThreadWatcher() {}

// static
void ThreadWatcher::StartWatching(const WatchingParams& params) {
  DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  DCHECK_GE(params.unresponsive_time.InMilliseconds(),
            params.sleep_time.InMilliseconds());

  // If we are not on WatchDogThread, then post a task to call StartWatching on
  // WatchDogThread.
  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
    WatchDogThread::PostTask(
        FROM_HERE,
        base::Bind(&ThreadWatcher::StartWatching, params));
    return;
  }

  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  // Create a new thread watcher object for the given thread and activate it.
  ThreadWatcher* watcher = new ThreadWatcher(params);

  DCHECK(watcher);
  // If we couldn't register the thread watcher object, we are shutting down,
  // then don't activate thread watching.
  if (!ThreadWatcherList::IsRegistered(params.thread_id))
    return;
  watcher->ActivateThreadWatching();
}

void ThreadWatcher::ActivateThreadWatching() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  if (active_) return;
  active_ = true;
  ping_count_ = unresponsive_threshold_;
  ResetHangCounters();
  base::MessageLoop::current()->PostTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::PostPingMessage,
                 weak_ptr_factory_.GetWeakPtr()));
}

void ThreadWatcher::DeActivateThreadWatching() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  active_ = false;
  ping_count_ = 0;
  weak_ptr_factory_.InvalidateWeakPtrs();
}

void ThreadWatcher::WakeUp() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // There is some user activity, PostPingMessage task of thread watcher if
  // needed.
  if (!active_) return;

  // Throw away the previous |unresponsive_count_| and start over again. Just
  // before going to sleep, |unresponsive_count_| could be very close to
  // |unresponsive_threshold_| and when user becomes active,
  // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
  // response for ping messages. Reset |unresponsive_count_| to start measuring
  // the unresponsiveness of the threads when system becomes active.
  unresponsive_count_ = 0;

  if (ping_count_ <= 0) {
    ping_count_ = unresponsive_threshold_;
    ResetHangCounters();
    PostPingMessage();
  } else {
    ping_count_ = unresponsive_threshold_;
  }
}

void ThreadWatcher::PostPingMessage() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // If we have stopped watching or if the user is idle, then stop sending
  // ping messages.
  if (!active_ || ping_count_ <= 0)
    return;

  // Save the current time when we have sent ping message.
  ping_time_ = base::TimeTicks::Now();

  // Send a ping message to the watched thread. Callback will be called on
  // the WatchDogThread.
  base::Closure callback(
      base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
                 ping_sequence_number_));
  if (watched_loop_->PostTask(
          FROM_HERE,
          base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
                     callback))) {
      // Post a task to check the responsiveness of watched thread.
      base::MessageLoop::current()->PostDelayedTask(
          FROM_HERE,
          base::Bind(&ThreadWatcher::OnCheckResponsiveness,
                     weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
          unresponsive_time_);
  } else {
    // Watched thread might have gone away, stop watching it.
    DeActivateThreadWatching();
  }
}

void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  // Record watched thread's response time.
  base::TimeTicks now = base::TimeTicks::Now();
  base::TimeDelta response_time = now - ping_time_;
  response_time_histogram_->AddTime(response_time);

  // Save the current time when we have got pong message.
  pong_time_ = now;

  // Check if there are any extra pings in flight.
  DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
  if (ping_sequence_number_ != ping_sequence_number)
    return;

  // Increment sequence number for the next ping message to indicate watched
  // thread is responsive.
  ++ping_sequence_number_;

  // If we have stopped watching or if the user is idle, then stop sending
  // ping messages.
  if (!active_ || --ping_count_ <= 0)
    return;

  base::MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::PostPingMessage,
                 weak_ptr_factory_.GetWeakPtr()),
      sleep_time_);
}

void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  // If we have stopped watching then consider thread as responding.
  if (!active_) {
    responsive_ = true;
    return;
  }
  // If the latest ping_sequence_number_ is not same as the ping_sequence_number
  // that is passed in, then we can assume OnPongMessage was called.
  // OnPongMessage increments ping_sequence_number_.
  if (ping_sequence_number_ != ping_sequence_number) {
    // Reset unresponsive_count_ to zero because we got a response from the
    // watched thread.
    ResetHangCounters();

    responsive_ = true;
    return;
  }
  // Record that we got no response from watched thread.
  GotNoResponse();

  // Post a task to check the responsiveness of watched thread.
  base::MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&ThreadWatcher::OnCheckResponsiveness,
                 weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
      unresponsive_time_);
  responsive_ = false;
}

void ThreadWatcher::Initialize() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  ThreadWatcherList::Register(this);

  const std::string response_time_histogram_name =
      "ThreadWatcher.ResponseTime." + thread_name_;
  response_time_histogram_ = base::Histogram::FactoryTimeGet(
      response_time_histogram_name,
      base::TimeDelta::FromMilliseconds(1),
      base::TimeDelta::FromSeconds(100), 50,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string unresponsive_time_histogram_name =
      "ThreadWatcher.Unresponsive." + thread_name_;
  unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
      unresponsive_time_histogram_name,
      base::TimeDelta::FromMilliseconds(1),
      base::TimeDelta::FromSeconds(100), 50,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string responsive_count_histogram_name =
      "ThreadWatcher.ResponsiveThreads." + thread_name_;
  responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
      responsive_count_histogram_name, 1, 10, 11,
      base::Histogram::kUmaTargetedHistogramFlag);

  const std::string unresponsive_count_histogram_name =
      "ThreadWatcher.UnresponsiveThreads." + thread_name_;
  unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
      unresponsive_count_histogram_name, 1, 10, 11,
      base::Histogram::kUmaTargetedHistogramFlag);
}

// static
void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
                                  const base::Closure& callback_task) {
  // This method is called on watched thread.
  DCHECK(BrowserThread::CurrentlyOn(thread_id));
  WatchDogThread::PostTask(FROM_HERE, callback_task);
}

void ThreadWatcher::ResetHangCounters() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  unresponsive_count_ = 0;
  hung_processing_complete_ = false;
}

void ThreadWatcher::GotNoResponse() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());

  ++unresponsive_count_;
  if (!IsVeryUnresponsive())
    return;

  // Record total unresponsive_time since last pong message.
  base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
  unresponsive_time_histogram_->AddTime(unresponse_time);

  // We have already collected stats for the non-responding watched thread.
  if (hung_processing_complete_)
    return;

  // Record how other threads are responding.
  uint32 responding_thread_count = 0;
  uint32 unresponding_thread_count = 0;
  ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
                                        &unresponding_thread_count);

  // Record how many watched threads are responding.
  responsive_count_histogram_->Add(responding_thread_count);

  // Record how many watched threads are not responding.
  unresponsive_count_histogram_->Add(unresponding_thread_count);

  // Crash the browser if the watched thread is to be crashed on hang and if the
  // number of other threads responding is less than or equal to
  // live_threads_threshold_ and at least one other thread is responding.
  if (crash_on_hang_ &&
      responding_thread_count > 0 &&
      responding_thread_count <= live_threads_threshold_) {
    static bool crashed_once = false;
    if (!crashed_once) {
      crashed_once = true;
      CrashBecauseThreadWasUnresponsive(thread_id_);
    }
  }

  hung_processing_complete_ = true;
}

bool ThreadWatcher::IsVeryUnresponsive() {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  return unresponsive_count_ >= unresponsive_threshold_;
}

// ThreadWatcherList methods and members.
//
// static
ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
// static
bool ThreadWatcherList::g_stopped_ = false;
// static
const int ThreadWatcherList::kSleepSeconds = 1;
// static
const int ThreadWatcherList::kUnresponsiveSeconds = 2;
// static
const int ThreadWatcherList::kUnresponsiveCount = 9;
// static
const int ThreadWatcherList::kLiveThreadsThreshold = 2;
// static, non-const for tests.
int ThreadWatcherList::g_initialize_delay_seconds = 120;

ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
    uint32 live_threads_threshold,
    uint32 unresponsive_threshold)
    : live_threads_threshold(live_threads_threshold),
      unresponsive_threshold(unresponsive_threshold) {
}

ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
    : live_threads_threshold(kLiveThreadsThreshold),
      unresponsive_threshold(kUnresponsiveCount) {
}

// static
void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
  uint32 unresponsive_threshold;
  CrashOnHangThreadMap crash_on_hang_threads;
  ParseCommandLine(command_line,
                   &unresponsive_threshold,
                   &crash_on_hang_threads);

  ThreadWatcherObserver::SetupNotifications(
      base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));

  WatchDogThread::PostTask(
      FROM_HERE,
      base::Bind(&ThreadWatcherList::SetStopped, false));

  if (!WatchDogThread::PostDelayedTask(
          FROM_HERE,
          base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
                     unresponsive_threshold,
                     crash_on_hang_threads),
          base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
    // Disarm() the startup timebomb, if we couldn't post the task to start the
    // ThreadWatcher (becasue WatchDog thread is not running).
    StartupTimeBomb::DisarmStartupTimeBomb();
  }
}

// static
void ThreadWatcherList::StopWatchingAll() {
  // TODO(rtenneti): Enable ThreadWatcher.
  ThreadWatcherObserver::RemoveNotifications();
  DeleteAll();
}

// static
void ThreadWatcherList::Register(ThreadWatcher* watcher) {
  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  if (!g_thread_watcher_list_)
    return;
  DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
  g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
}

// static
bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
{'fraction_non_alphanumeric': 0.09818533300074832, 'fraction_numerical': 0.004209279121975555, 'mean_word_length': 4.5576156645295445, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386437', 'n_tokens_mistral': 9951, 'n_tokens_neox': 9382, 'n_words': 2086}