RP reports `timeout` even when pilot terminated properly
See radical-collaboration/Glatard-Lab/issues/1 for example.  The `timeout` message is confusing and wrong.
diff --git a/src/radical/pilot/agent/agent_0.py b/src/radical/pilot/agent/agent_0.py
index e7978874b..21e4833c8 100755
--- a/src/radical/pilot/agent/agent_0.py
+++ b/src/radical/pilot/agent/agent_0.py
@@ -73,6 +73,7 @@ class Agent_0(rpu.Worker):
         # and start those
         self._cmgr = rpu.ComponentManager(self._cfg)
         self._cfg.heartbeat = self._cmgr.cfg.heartbeat
+
         self._cmgr.start_bridges()
         self._cmgr.start_components()
 
@@ -227,6 +228,8 @@ class Agent_0(rpu.Worker):
                     self._log.error('output tarring failed: %s', cmd)
 
 
+    # --------------------------------------------------------------------------
+    #
     def finalize(self):
 
         # tar up output staging data
@@ -248,59 +251,31 @@ class Agent_0(rpu.Worker):
         # we don't rely on the existence / viability of the update worker at
         # that point.
         self._log.debug('update db state: %s: %s', state, self._final_cause)
-        self._update_db(state, self._final_cause)
 
         # NOTE: we do not push the final pilot state, as that is done by the
         #       bootstrapper *after* this pilot *actually* finished.
+        with open('./killme.signal', 'w') as fout:
+            fout.write('%s\n' % state)
 
         self._log.info('pilot state: %s [%s]', state, self._final_cause)
-
-        out, out, err = None, None, None
-
-        try   : out = open('./agent.0.out', 'r').read(1024)
-        except: pass
-        try   : err = open('./agent.0.err', 'r').read(1024)
-        except: pass
-        try   : log = open('./agent.0.log', 'r').read(1024)
-        except: pass
-
-        self._dbs._c.update({'type': 'pilot',
-                             'uid' : self._pid},
-                            {'$set': {'stdout' : rpu.tail(out),
-                                      'stderr' : rpu.tail(err),
-                                      'logfile': rpu.tail(log)} })
-
-
-    # --------------------------------------------------------------------------
-    #
-    def _update_db(self, state, msg=None):
-
-        # NOTE: we do not push the final pilot state, as that is done by the
-        #       bootstrapper *after* this poilot *actually* finished.
-
-        self._log.info('pilot state: %s', state)
         self._log.info('rusage: %s', rpu.get_rusage())
-        self._log.info(msg)
-
-        if state == rps.FAILED:
-            self._log.info(ru.get_trace())
 
-        out = None
-        err = None
-        log = None
+        out, out, err = '', '', ''
 
-        try   : out = open('./agent_0.out', 'r').read(1024)
+        try   : out   = open('./agent.0.out', 'r').read(1024)
         except: pass
-        try   : err = open('./agent_0.err', 'r').read(1024)
+        try   : err   = open('./agent.0.err', 'r').read(1024)
         except: pass
-        try   : log = open('./agent_0.log', 'r').read(1024)
+        try   : log   = open('./agent.0.log', 'r').read(1024)
         except: pass
 
-        ret = self._dbs._c.update({'type': 'pilot',
-                                   'uid' : self._pid},
-                                  {'$set': {'stdout' : rpu.tail(out),
-                                            'stderr' : rpu.tail(err),
-                                            'logfile': rpu.tail(log)}
+        ret = self._dbs._c.update({'type' : 'pilot',
+                                   'uid'  : self._pid},
+                                  {'$set' : {'stdout' : rpu.tail(out),
+                                             'stderr' : rpu.tail(err),
+                                             'logfile': rpu.tail(log),
+                                             'state'  : state},
+                                   '$push': {'states' : state}
                                   })
         self._log.debug('update ret: %s', ret)
 
diff --git a/src/radical/pilot/agent/bootstrap_0.sh b/src/radical/pilot/agent/bootstrap_0.sh
index d759e0596..9e9c95bb3 100755
--- a/src/radical/pilot/agent/bootstrap_0.sh
+++ b/src/radical/pilot/agent/bootstrap_0.sh
@@ -1813,6 +1813,7 @@ do
     sleep 3
     if kill -0 $AGENT_PID 2>/dev/null
     then
+        echo -n '.'
         if test -e "./killme.signal"
         then
             profile_event 'killme' "`date --rfc-3339=ns | cut -c -23`"
@@ -1827,6 +1828,7 @@ do
             kill  -9 $AGENT_PID
         fi
     else
+        echo
         profile_event 'agent_gone' "`date --rfc-3339=ns | cut -c -23`"
         echo "agent $AGENT_PID is gone"
         break
diff --git a/src/radical/pilot/pilot_manager.py b/src/radical/pilot/pilot_manager.py
index 7c4f4a984..7b49ba345 100644
--- a/src/radical/pilot/pilot_manager.py
+++ b/src/radical/pilot/pilot_manager.py
@@ -190,7 +190,6 @@ class PilotManager(rpu.Component):
         if self._closed:
             return
 
-        self._terminate.set()
         self._rep.info('<<close pilot manager')
 
         # disable callbacks during shutdown
@@ -201,11 +200,12 @@ class PilotManager(rpu.Component):
 
         # If terminate is set, we cancel all pilots.
         if terminate:
-            self.cancel_pilots(_timeout=10)
+            self.cancel_pilots(_timeout=20)
             # if this cancel op fails and the pilots are s till alive after
             # timeout, the pmgr.launcher termination will kill them
 
 
+        self._terminate.set()
         self._cmgr.close()
 
         self._log.info("Closed PilotManager %s." % self._uid)
@@ -717,8 +717,7 @@ class PilotManager(rpu.Component):
                     self._log.debug ("wait timed out")
                     break
 
-                time.sleep (0.1)
-
+            time.sleep (0.1)
 
         self._rep.idle(mode='stop')
 
diff --git a/src/radical/pilot/pmgr/launching/default.py b/src/radical/pilot/pmgr/launching/default.py
index 3551d69bc..d864d109e 100644
--- a/src/radical/pilot/pmgr/launching/default.py
+++ b/src/radical/pilot/pmgr/launching/default.py
@@ -697,7 +697,7 @@ class Default(PMGRLaunchingComponent):
             tgt_dir = os.path.dirname(tgt)
 
             if tgt_dir.startswith('..'):
-              # raise ValueError('staging target %s outside of pilot sandbox: %s' % (ft['tgt'], tgt))
+              # raise ValueError('staging tgt %s outside pilot sbox: %s' % (ft['tgt'], tgt))
                 tgt = ft['tgt']
                 tgt_dir = os.path.dirname(tgt)
 
['src/radical/pilot/agent/bootstrap_0.sh', 'src/radical/pilot/pilot_manager.py', 'src/radical/pilot/agent/agent_0.py', 'src/radical/pilot/pmgr/launching/default.py']