File permission err for non-default .radical/ path in ptyshell
Hi ! 

I'm working on a web interface wrapping saga but I have configuration problem.
I'm running my app with django on apache. 
Without any conf, my home is /var/www/ , so /var/www/.radical/*  raise permission error
I tried to set RADICAL_BASE and it partially worked, as well as RADICAL_SAGA_LOG_TGT in /var/www/html/<my_project>

But when I tried to initiate a shell connection I got an error about /var/www/.radical/ again.
With further tracing I've ended finding this : 

https://github.com/radical-cybertools/radical.saga/blob/ed6634ff8d7b31ff02b5f23d2aad1189d0f73360/src/radical/saga/utils/pty_shell.py#L228-L230

Is it possible to override this ? Maybe using conf file ?

Besides, is there a way to display all configs values ?


diff --git a/examples/context/context_ssh.py b/examples/context/context_ssh.py
index e4b054c1..b9c89814 100755
--- a/examples/context/context_ssh.py
+++ b/examples/context/context_ssh.py
@@ -5,31 +5,24 @@ __copyright__ = "Copyright 2012-2013, The SAGA Project"
 __license__   = "MIT"
 
 
-import sys
+import os
 
 import radical.saga as rs
 
 
-def main():
+# ------------------------------------------------------------------------------
+#
 
-    try:
+c = rs.Context ('ssh')
+c.user_id   = os.environ['USER']
+c.user_cert = '%s/.ssh/id_rsa' % os.environ['HOME']
 
-        c = rs.Context ('ssh')
-        c.user_id   = 'tg12736'
-        c.user_cert = '/home/user/ssh/id_rsa_xsede' # private key derived from cert
+s = rs.Session (default=False)  # create session with no contexts
+s.add_context (c)
 
-        s = rs.Session (default=False)            # create session with no contexts
-        s.add_context (c)
+js = rs.job.Service ('ssh://localhost', session=s)
+js.run_job ("/bin/true")
 
-        js = rs.job.Service ('ssh://login1.stampede.tacc.utexas.edu', session=s)
-        js.run_job ("/bin/true")
 
-    except rs.SagaException as ex:
-        # Catch all saga exceptions
-        print("An exception occured: (%s) %s " % (ex.type, (str(ex))))
-        # Trace back the exception. That can be helpful for debugging.
-        print(" \n*** Backtrace:\n %s" % ex.traceback)
-        return -1
+# ------------------------------------------------------------------------------
 
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/src/radical/saga/adaptors/cobalt/cobaltjob.py b/src/radical/saga/adaptors/cobalt/cobaltjob.py
index d96b3d3a..26e5f0df 100644
--- a/src/radical/saga/adaptors/cobalt/cobaltjob.py
+++ b/src/radical/saga/adaptors/cobalt/cobaltjob.py
@@ -23,6 +23,8 @@ from ..               import base       as a_base
 from ..cpi            import job        as cpi_job
 from ..cpi            import decorators as cpi_decs
 
+import radical.utils as ru
+
 
 SYNC_CALL  = cpi_decs.SYNC_CALL
 ASYNC_CALL = cpi_decs.ASYNC_CALL
@@ -308,16 +310,6 @@ _PTY_TIMEOUT = 2.0
 _ADAPTOR_NAME          = "radical.saga.adaptors.cobaltjob"
 _ADAPTOR_SCHEMAS       = ["cobalt", "cobalt+ssh", "cobalt+gsissh"]
 _ADAPTOR_OPTIONS       = [
-    {
-        'category'         : 'radical.saga.adaptors.cobaltjob',
-        'name'             : 'base_workdir',
-        'type'             : str,
-        'default'          : "$HOME/.radical/saga/adaptors/cobaltjob/",
-        'documentation'    : '''The adaptor stores job state information on the
-                              filesystem on the target resource. This parameter
-                              specified what location should be used.''',
-        'env_variable'     : None
-    },
     {
         'category'         : 'radical.saga.adaptors.cobaltjob',
         'name'             : 'purge_on_start',
@@ -434,11 +426,10 @@ class Adaptor (a_base.Base):
         self.epoch = datetime.datetime(1970,1,1)
 
         # Adaptor Options
-        self.base_workdir     = self._cfg['base_workdir']
         self.purge_on_start   = self._cfg['purge_on_start']
         self.purge_older_than = self._cfg['purge_older_than']
 
-        self.base_workdir     = os.path.normpath(self.base_workdir)
+        self.base_workdir = ru.get_radical_base('saga') + 'adaptors/cobalt'
 
         # dictionaries to keep track of certain Cobalt jobs data
         self._script_file         = dict()  # location of cobalt script file
@@ -575,8 +566,9 @@ class CobaltJobService (cpi_job.Service):
     def initialize(self):
 
         # Create the staging directory
-        ret, out, _ = self.shell.run_sync ("mkdir -p %s"
-                                                   % self._adaptor.base_workdir)
+        ret, out, _ = self.shell.run_sync("mkdir -p %s"
+                                          % self._adaptor.base_workdir)
+
         if  ret != 0 :
             raise rse.NoSuccess("Error creating staging directory. (%s): (%s)"
                            % (ret, out))
diff --git a/src/radical/saga/adaptors/context/myproxy.py b/src/radical/saga/adaptors/context/myproxy.py
index 6d4740be..33a551fb 100644
--- a/src/radical/saga/adaptors/context/myproxy.py
+++ b/src/radical/saga/adaptors/context/myproxy.py
@@ -14,6 +14,8 @@ from ..cpi         import context as cpi
 from ...           import context as api
 from ...exceptions import *
 
+import radical.utils as ru
+
 
 ######################################################################
 #
@@ -66,7 +68,7 @@ class Adaptor (base.Base):
 
         # there are no default myproxy contexts
         self._default_contexts = []
-        self.base_workdir = self._cfg.get('base_workdir', os.getcwd())
+        self.base_workdir = ru.get_radical_base('saga') + 'adaptors/myproxy'
 
 
     def sanity_check (self) :
@@ -89,6 +91,7 @@ class ContextMyProxy (cpi.Context) :
 
         _cpi_base = super  (ContextMyProxy, self)
         _cpi_base.__init__ (api, adaptor)
+
         self.base_workdir = adaptor.base_workdir
 
 
@@ -138,11 +141,10 @@ class ContextMyProxy (cpi.Context) :
         proxy_store    = "%s" % self.base_workdir
         proxy_location = "%s/myproxy_%d.x509"  %  (proxy_store, id(self))
 
-        if not os.path.exists (proxy_store) :
-            try :
-                os.makedirs (proxy_store)
-            except OSError as e :
-                raise NoSuccess ("could not create myproxy store") from e
+        try:
+            ru.rec_makedir(proxy_store)
+        except OSError as e:
+            raise NoSuccess('could not create myproxy store') from e
 
         cmd += " --out %s"  %  proxy_location
 
diff --git a/src/radical/saga/adaptors/loadl/loadljob.py b/src/radical/saga/adaptors/loadl/loadljob.py
index d49eb597..e4139a13 100644
--- a/src/radical/saga/adaptors/loadl/loadljob.py
+++ b/src/radical/saga/adaptors/loadl/loadljob.py
@@ -233,7 +233,7 @@ class LOADLJobService (cpi.job.Service):
         self.enforce_consumable_memory = False
         self.enforce_consumable_virtual_memory = False
         self.enforce_consumable_large_page_memory = False
-        self.temp_path = "$HOME/.radical/saga/adaptors/loadl_job"
+        self.temp_path = ru.get_radical_base('saga') + 'adaptors/loadl_job'
 
         # LoadLeveler has two ways of specifying the executable and arguments.
         # - Explicit: the executable and arguments are specified as parameters.
@@ -354,9 +354,9 @@ class LOADLJobService (cpi.job.Service):
 
         # purge temporary files
         if self._adaptor.purge_on_start:
-            cmd = "find $HOME/.radical/saga/adaptors/loadl_job" \
-                  " -type f -mtime +%d -print -delete | wc -l" \
-                % self._adaptor.purge_older_than
+            cmd  = "find %s" % self.temp_path
+            cmd += " -type f -mtime +%d -print -delete | wc -l" \
+                                               % self._adaptor.purge_older_than
             ret, out, _ = self.shell.run_sync(cmd)
             if ret == 0 and out != "0":
                 self._logger.info("Purged %s temporary files" % out)
diff --git a/src/radical/saga/adaptors/sge/sgejob.py b/src/radical/saga/adaptors/sge/sgejob.py
index 0693b453..6158d5eb 100644
--- a/src/radical/saga/adaptors/sge/sgejob.py
+++ b/src/radical/saga/adaptors/sge/sgejob.py
@@ -217,7 +217,8 @@ class Adaptor (a_base.Base):
 
         self.purge_on_start   = self._cfg['purge_on_start']
         self.purge_older_than = self._cfg['purge_older_than']
-        self.base_workdir     = self._cfg['base_workdir']
+
+        self.base_workdir = ru.get_radical_base('saga') + 'adaptors/sge'
 
 
     # ----------------------------------------------------------------
diff --git a/src/radical/saga/adaptors/shell/shell_job.py b/src/radical/saga/adaptors/shell/shell_job.py
index 91b9fcd9..81408f79 100644
--- a/src/radical/saga/adaptors/shell/shell_job.py
+++ b/src/radical/saga/adaptors/shell/shell_job.py
@@ -263,7 +263,7 @@ _ADAPTOR_DOC           = {
           * number of files are limited, as is disk space: the job.service will
 
             keep job state on the remote disk, in
-            ``~/.radical/saga/adaptors/shell_job/``.
+            `$RADICAL_BASE/.radical/saga/adaptors/shell_job/`.
             Quota limitations may limit the number of files created there,
             and/or the total size of that directory.
 
@@ -557,8 +557,8 @@ class ShellJobService(cpi.Service):
         # expand those config entries we want to use(where needed)
         self.notifications  = cfg.enable_notifications
         self.purge_on_start = cfg.purge_on_star
-        self.base_workdir   = ru.expand_env(cfg.base_workdir, env)
 
+        self.base_workdir   = ru.get_radical_base('saga') + 'adaptors/shell_job'
 
         # start the shell, find its prompt.  If that is up and running, we can
         # bootstrap our wrapper script, and then run jobs etc.
@@ -568,12 +568,11 @@ class ShellJobService(cpi.Service):
         # stdio.
 
         base = self.base_workdir
-
         with self._shell_lock:
-            ret, out, _ = self.shell.run_sync(" mkdir -p %s" % base)
+            ret, out, _ = self.shell.run_sync(' mkdir -p %s' % base)
 
         if ret != 0:
-            raise rse.NoSuccess("host setup failed(%s):(%s)" % (ret, out))
+            raise rse.NoSuccess('host setup failed (%s):(%s)' % (ret, out))
 
         # TODO: replace some constants in the script with values from config
         # files, such as 'timeout' or 'purge_on_quit' ...
diff --git a/src/radical/saga/adaptors/shell/shell_wrapper.sh b/src/radical/saga/adaptors/shell/shell_wrapper.sh
index 937e5608..52459da9 100644
--- a/src/radical/saga/adaptors/shell/shell_wrapper.sh
+++ b/src/radical/saga/adaptors/shell/shell_wrapper.sh
@@ -10,8 +10,8 @@ export HISTIGNORE
 #
 # The invocation passes one (optional) parameter, the base workdir.  That
 # directory will be used to keep job state data. Its default value is set to
-# $HOME/.radical/saga/adaptors/shell_job/
-
+# $RADICAL_BASE/.radical/saga/adaptors/shell_job/ (defaults to $HOME)
+test -z "$RADICAL_BASE" && export RADICAL_BASE="$HOME"
 
 # --------------------------------------------------------------------
 # on argument quoting
@@ -69,7 +69,7 @@ export GID
 BASE="$1"; shift
 if test -z "$BASE"
 then
-  BASE=$HOME/.radical/saga/adaptors/shell_job/
+  BASE=$RADICAL_BASE/.radical/saga/adaptors/shell_job/
 fi
 NOTIFICATIONS="$BASE/notifications"
 LOG="$BASE/log"
diff --git a/src/radical/saga/configs/adaptors_cobaltjob.json b/src/radical/saga/configs/adaptors_cobaltjob.json
index 59d3bb6a..96fc313f 100644
--- a/src/radical/saga/configs/adaptors_cobaltjob.json
+++ b/src/radical/saga/configs/adaptors_cobaltjob.json
@@ -1,8 +1,5 @@
 
 {
-    # where to store state
-    "base_workdir"     : "${HOME}/.radical/saga/adaptors/cobalt_job/}",
-
     # purge old state on start
     "purge_on_start"   : false,
 
diff --git a/src/radical/saga/configs/adaptors_myproxy.json b/src/radical/saga/configs/adaptors_myproxy.json
index 770d578f..0e630511 100644
--- a/src/radical/saga/configs/adaptors_myproxy.json
+++ b/src/radical/saga/configs/adaptors_myproxy.json
@@ -1,9 +1,5 @@
 
 {
-    "enabled"      : "True",
-
-    # The adaptor stores proxy information on the filesystem on the local
-    # resource.  This parameter specified what location should be used.
-    "base_workdir" : "${HOME:.}/.radical/saga/adaptors/proxies/"
+    "enabled"      : "True"
 }
 
diff --git a/src/radical/saga/configs/adaptors_sgejob.json b/src/radical/saga/configs/adaptors_sgejob.json
index 79439839..9d471559 100644
--- a/src/radical/saga/configs/adaptors_sgejob.json
+++ b/src/radical/saga/configs/adaptors_sgejob.json
@@ -1,15 +1,11 @@
 
-{ 
+{
     # Purge temporary job information for all jobs which are older than a number
     # of days.  The number of days can be configured with <purge_older_than>.
     "purge_on_start" : true,
 
     # When <purge_on_start> is enabled this specifies the number of days to
     # consider a temporary file older enough to be deleted.
-    "purge_older_than" : 30, 
-
-    # The adaptor stores job state information on the filesystem on the target
-    # resource.  This parameter specified what location should be used.
-    "base_workdir" : "${HOME}/.radical/saga/adaptors/sge_job/"
+    "purge_older_than" : 30
 }
 
diff --git a/src/radical/saga/configs/adaptors_shell_job.json b/src/radical/saga/configs/adaptors_shell_job.json
index 03588d4b..efbe81f3 100644
--- a/src/radical/saga/configs/adaptors_shell_job.json
+++ b/src/radical/saga/configs/adaptors_shell_job.json
@@ -11,10 +11,6 @@
     # state when starting the job service instance. Note that this will purge
     # *all* suitable jobs, including the ones managed by another, live job
     # service instance.
-    "purge_on_start" : true,
-
-    # The adaptor stores job state information on the filesystem on the target
-    # resource.  This parameter specified what location should be used.
-    "base_workdir" : "${HOME}/.radical/saga/adaptors/shell_job/"
+    "purge_on_start" : true
 }
 
diff --git a/src/radical/saga/utils/pty_shell.py b/src/radical/saga/utils/pty_shell.py
index fb12e19e..ead78b3e 100644
--- a/src/radical/saga/utils/pty_shell.py
+++ b/src/radical/saga/utils/pty_shell.py
@@ -133,7 +133,7 @@ class PTYShell (object) :
     that purpose, the shell started on the first channel will create a named
     pipe, at::
 
-      $HOME/.radical/saga/adaptors/shell/async.$$
+      $RADICAL_BASE/.radical/saga/adaptors/shell/async.$$
 
     ``$$`` here represents the pid of the shell process.  It will also set the
     environment variable ``SAGA_ASYNC_PIPE`` to point to that named pipe -- any
@@ -225,20 +225,12 @@ class PTYShell (object) :
         self.prompt_re = re.compile ("^(.*?)%s"    % self.prompt, re.DOTALL)
         self.logger.info ("PTY prompt pattern: %s" % self.prompt)
 
-        # we need a local dir for file staging caches.  At this point we use
-        # $HOME, but should make this configurable (FIXME)
-        self.base = os.environ['HOME'] + '/.radical/saga/adaptors/shell/'
-
+        # local dir for file staging caches
+        self.base = ru.get_radical_base('saga') + 'adaptors/shell/'
         try:
-            os.makedirs (self.base)
-
+            ru.rec_makedir(self.base)
         except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir (self.base):
-                pass
-            else:
-                raise rse.NoSuccess ("could not create staging dir: %s" % e) \
-                      from e
-
+            raise rse.NoSuccess('could not create staging dir: %s' % e) from e
 
         self.factory    = supsf.PTYShellFactory   ()
         self.pty_info   = self.factory.initialize (self.url,    self.session,
['src/radical/saga/adaptors/loadl/loadljob.py', 'src/radical/saga/configs/adaptors_shell_job.json', 'src/radical/saga/adaptors/shell/shell_job.py', 'src/radical/saga/configs/adaptors_myproxy.json', 'src/radical/saga/adaptors/context/myproxy.py', 'src/radical/saga/adaptors/cobalt/cobaltjob.py', 'src/radical/saga/adaptors/shell/shell_wrapper.sh', 'src/radical/saga/configs/adaptors_cobaltjob.json', 'src/radical/saga/configs/adaptors_sgejob.json', 'src/radical/saga/utils/pty_shell.py', 'src/radical/saga/adaptors/sge/sgejob.py', 'examples/context/context_ssh.py']