Deadlock in token expiration
Motivation
----------
In each token request the authentication core removes all already expired tokens of the account. For accounts (e.g. panda, pilot) with many tokens, this currently leads to deadlocks. It is not clear if there is some query degradation, this is currently being investigated. This is the first time we are encountering this issue.

To circumvent this issue, we currently disabled the token expiration. 
https://github.com/rucio/rucio/blob/342e0831ca4088dbb3b4f256133469b8b85c5d8c/lib/rucio/core/authentication.py#L152
The tokens lifetime is correctly validated, so this only leads to tokens being archived for longer.

We should change this expiration to use `for_update(skip_locked=True)` or do the token cleanup entirely in a different process.
diff --git a/lib/rucio/core/authentication.py b/lib/rucio/core/authentication.py
index 98baf238c..564081b31 100644
--- a/lib/rucio/core/authentication.py
+++ b/lib/rucio/core/authentication.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario@lassnig.net>, 2012-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2012-2017
+# - Martin Barisits <martin.barisits@cern.ch>, 2012-2019
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2017
 # - Angelos Molfetas <Angelos.Molfetas@cern.ch>, 2012
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
@@ -114,7 +114,7 @@ def get_auth_token_user_pass(account, username, password, appid, ip=None, sessio
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -150,7 +150,7 @@ def get_auth_token_x509(account, dn, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -186,7 +186,7 @@ def get_auth_token_gss(account, gsstoken, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -250,7 +250,7 @@ def get_auth_token_ssh(account, signature, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
['lib/rucio/core/authentication.py']