private boolean resolveHang ( long tasksCompleted , boolean queueEmpty , int poolSize ) { boolean actionTaken = false ; if ( tasksCompleted == 0 && ! queueEmpty ) { /** * When a hang is detected the controller enters hang resolution mode. * The controller will run on a shorter-than-usual cycle for hangResolutionCycles * from the last hang detection, to resolve hang situations more quickly. */ if ( hangResolutionCountdown == 0 ) { // cancel regular controller schedule activeTask . cancel ( ) ; // restart with shortened interval for quicker hang resolution activeTask = new IntervalTask ( this ) ; timer . schedule ( activeTask , hangInterval , hangInterval ) ; } hangResolutionCountdown = hangResolutionCycles ; controllerCyclesWithoutHang = 0 ; // if this is the first time we detected a given deadlock, record how many threads there are // and print a message if ( poolSizeWhenHangDetected < 0 ) { poolSizeWhenHangDetected = poolSize ; if ( tc . isEventEnabled ( ) ) { Tr . event ( tc , "Executor hang detected at poolSize=" + poolSizeWhenHangDetected , threadPool ) ; } } else if ( tc . isEventEnabled ( ) ) { Tr . event ( tc , "Executor hang continued at poolSize=" + poolSize , threadPool ) ; } setPoolIncrementDecrement ( poolSize ) ; if ( poolSize + poolIncrement <= maxThreads && poolSize < MAX_THREADS_TO_BREAK_HANG ) { targetPoolSize = adjustPoolSize ( poolSize , poolIncrement ) ; // update the poolSize set to resolve the hang, plus a buffer amount int targetSize = poolSize + ( compareRange * poolIncrement ) ; if ( hangBufferPoolSize < targetSize ) { hangBufferPoolSize = targetSize ; } actionTaken = true ; } else { // there's a hang, but we can't add any more threads... emit a warning the first time this // happens for a given hang, but otherwise just bail if ( hangMaxThreadsMessageEmitted == false && hangIntervalCounter > 0 ) { if ( tc . isWarningEnabled ( ) ) { Tr . warning ( tc , "unbreakableExecutorHang" , poolSizeWhenHangDetected , poolSize ) ; } hangMaxThreadsMessageEmitted = true ; } } hangIntervalCounter ++ ; } else { // no hang exists, so reset the appropriate variables that track hangs poolSizeWhenHangDetected = - 1 ; hangIntervalCounter = 0 ; hangMaxThreadsMessageEmitted = false ; // manage hang resolution mode if ( hangResolutionCountdown > 0 ) { hangResolutionCountdown -- ; if ( hangResolutionCountdown <= 0 ) { // move out of hang resolution cycle time activeTask . cancel ( ) ; // restart using regular cycle time activeTask = new IntervalTask ( this ) ; timer . schedule ( activeTask , interval , interval ) ; } } /** * if controller is running below hangResolutionPoolSize marker without hanging, * we can reduce that marker ... the workload must have changed, so prior hang * information is no longer valid. We will reduce it gradually, to maintain a * conservative stance toward avoiding hangs. */ if ( hangBufferPoolSize > coreThreads ) { if ( hangBufferPoolSize > poolSize ) { controllerCyclesWithoutHang ++ ; if ( controllerCyclesWithoutHang > noHangCyclesThreshold ) { setPoolIncrementDecrement ( poolSize ) ; hangBufferPoolSize -= poolDecrement ; controllerCyclesWithoutHang = 0 ; } } } } return actionTaken ; }
Detects a hang in the underlying executor . When a hang is detected increases the poolSize in hopes of relieving the hang unless poolSize has reached maxThreads .