Workflow without reqmgr_workload spec file in couchdb
@vlimant reported failures in one of the unified modules and the apparent problem is that the workflow was created, with the proper reqmgr_workload_cache request doc in the database (couchdb)
https://cmsweb.cern.ch/reqmgr2/data/request?name=pdmvserv_task_SUS-RunIISummer16NanoAODv4-00040__v1_T_190124_181348_3206

but somehow the spec file for the same workflow is missing:
https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/pdmvserv_task_SUS-RunIISummer16NanoAODv4-00040__v1_T_190124_181348_3206/spec

The request status is New, and since it's related to couch, we can't have an atomic operation. However, we should likely invert the order in which these documents get persisted into the database. Having a spec file in there doesn't hurt, but the inverse does.

One more thing, the fetch API fails to find this same workload_cache because it tries to load the spec file for the splitting view
https://cmsweb.cern.ch/reqmgr2/fetch?rid=pdmvserv_task_SUS-RunIISummer16NanoAODv4-00040__v1_T_190124_181348_3206

To be investigated whether the same happened with workflows in other status...
diff --git a/src/python/WMCore/ReqMgr/Service/Request.py b/src/python/WMCore/ReqMgr/Service/Request.py
index 3f78bdee5..9ed426d71 100644
--- a/src/python/WMCore/ReqMgr/Service/Request.py
+++ b/src/python/WMCore/ReqMgr/Service/Request.py
@@ -557,9 +557,16 @@ class Request(RESTEntity):
             self._update_additional_request_args(workload, request_args)
 
             cherrypy.log("Create request, input args: %s ..." % request_args)
-            workload.saveCouch(request_args["CouchURL"], request_args["CouchWorkloadDBName"],
-                               metadata=request_args)
-            out.append({'request': workload.name()})
+            try:
+                workload.saveCouch(request_args["CouchURL"], request_args["CouchWorkloadDBName"],
+                                   metadata=request_args)
+                out.append({'request': workload.name()})
+            except Exception as ex:
+                # then it failed to add the spec file as attachment
+                # we better delete the original request to avoid confusion in wmstats
+                cherrypy.log("Error saving request spec to couch: %s " % str(ex))
+                self.delete(request_args['RequestName'])
+
         return out
 
 
diff --git a/src/python/WMCore/WMSpec/Persistency.py b/src/python/WMCore/WMSpec/Persistency.py
index 6e89c8bdb..2a08739a4 100644
--- a/src/python/WMCore/WMSpec/Persistency.py
+++ b/src/python/WMCore/WMSpec/Persistency.py
@@ -80,28 +80,33 @@ class PersistencyHelper:
         return
 
 
-    def saveCouch(self, couchUrl, couchDBName, metadata={}):
+    def saveCouch(self, couchUrl, couchDBName, metadata=None):
         """ Save this spec in CouchDB.  Returns URL """
-        import WMCore.Database.CMSCouch
-        server = WMCore.Database.CMSCouch.CouchServer(couchUrl)
+        from WMCore.Database.CMSCouch import CouchServer, CouchInternalServerError
+        metadata = metadata or {}
+        server = CouchServer(couchUrl)
         database = server.connectDatabase(couchDBName)
-        uri = '/%s/%s' % (couchDBName, self.name())
-        specuri = uri + '/spec'
         name = self.name()
-        if not database.documentExists(self.name()):
+        uri = '/%s/%s' % (couchDBName, name)
+        specuri = uri + '/spec'
+        if not database.documentExists(name):
             self.setSpecUrl(couchUrl + specuri)
             doc = database.put(uri, data=metadata, contentType='application/json')
             #doc = database.commitOne(self.name(), metadata)
             rev = doc['rev']
         else:
             #doc = database.get(uri+'?revs=true')
-            doc = database.document(self.name())
+            doc = database.document(name)
             rev = doc['_rev']
 
         #specuriwrev = specuri + '?rev=%s' % rev
         workloadString = pickle.dumps(self.data)
         #result = database.put(specuriwrev, workloadString, contentType='application/text')
-        result = database.addAttachment(name, rev, workloadString, 'spec')
+        retval = database.addAttachment(name, rev, workloadString, 'spec')
+        if retval.get('ok', False) is not True:
+            msg = "Failed to save a spec attachment in CouchDB for %s" % name
+            raise CouchInternalServerError(msg, data=None, result=retval)
+
         url = couchUrl + specuri
         return url
 
['src/python/WMCore/WMSpec/Persistency.py', 'src/python/WMCore/ReqMgr/Service/Request.py']