@ Override public void run ( ) { Thread . currentThread ( ) . setName ( "Latency Watchdog" ) ; LOG . info ( String . format ( "Latency Watchdog enabled -- threshold:%d(ms) " + "wakeup_interval:%d(ms) min_log_interval:%d(ms)\n" , WATCHDOG_THRESHOLD , WAKEUP_INTERVAL , MIN_LOG_INTERVAL ) ) ; while ( true ) { for ( Entry < Thread , AtomicLong > entry : sLatencyMap . entrySet ( ) ) { Thread t = entry . getKey ( ) ; long timestamp = entry . getValue ( ) . get ( ) ; long now = System . currentTimeMillis ( ) ; if ( ( now - timestamp > WATCHDOG_THRESHOLD ) && t . getState ( ) != Thread . State . TERMINATED ) { StringBuilder sb = new StringBuilder ( ) ; String format = t . getName ( ) + " has been delayed for more than " + WATCHDOG_THRESHOLD + " milliseconds\n %s" ; for ( StackTraceElement ste : t . getStackTrace ( ) ) { sb . append ( ste ) ; sb . append ( "\n" ) ; } RateLimitedLogger . tryLogForMessage ( now , MIN_LOG_INTERVAL , TimeUnit . MILLISECONDS , LOG , Level . INFO , format , sb . toString ( ) ) ; } } try { Thread . sleep ( WAKEUP_INTERVAL ) ; } catch ( Exception e ) { e . printStackTrace ( ) ; } } }
The watchdog thread will be invoked every WAKEUP_INTERVAL time to check if any thread that be monitored has not updated its time stamp more than WATCHDOG_THRESHOLD millisecond . Same stack trace messages are rate limited by MIN_LOG_INTERVAL .