Updating a cluster leads to API server downtime
First observation: the ELB healthcheck leads to ASG terminating a new instance because the `HealthCheckGracePeriod` defaults to 5 minutes and the master node sometimes takes longer than 5 minutes to come up :disappointed: 

diff --git a/cluster/cluster.py b/cluster/cluster.py
index 4acc7075..57a71e54 100755
--- a/cluster/cluster.py
+++ b/cluster/cluster.py
@@ -344,13 +344,20 @@ def drain_node(node_name: str, config: dict, max_grace_period=60):
     # respect pod terminate grace period
     grace_period = min(longest_grace_period(node_name, config), max_grace_period)
 
-    subprocess.check_call([
-        'kubectl',
-        '--server', config["api_server"],
-        '--token', config["worker_shared_secret"],
-        'drain', node_name,
-        '--force', '--delete-local-data',
-        '--grace-period={}'.format(grace_period)])
+    for i in range(3):
+        try:
+            subprocess.check_call([
+                'kubectl',
+                '--server', config["api_server"],
+                '--token', config["worker_shared_secret"],
+                'drain', node_name,
+                '--force', '--delete-local-data',
+                '--ignore-daemonsets',
+                '--grace-period={}'.format(grace_period)])
+            break
+        except Exception as e:
+            warning('Kubectl failed to drain node: {}'.format(e))
+            time.sleep(grace_period)
     time.sleep(grace_period)
 
 
['cluster/cluster.py']