Sorting schema string snag
The `de_sources_manual` parameter in ddcal worker is defined in the schema as:

```
         de_sources_manual:
           seq:
             - type: str
           desc: List of sources per target to tag for DD calibration, in the same order as the 'de_target_manual' list. Use ';' to separate different sources per target.
           required: false
           example: ''
```

It took the list of sources in the format ['RA1,DEC1,SIZE1'; 'RA2, DEC2,SIZE2';......], a list of strings with 1st element 'RA1,DEC1,SIZE1' and so on; everything was fine in the paradise. Recently, however, CARACal has started the format to mean  a list formed of elements 'RA1', 'RA2', 'SIZE1', 'RA2'.... , which plays havoc with the ddcal worker code. If I use ";" instead of "," it does take it as a single source, but remember that ";" is already use to separate different sources, so that's a dead end.

@o-smirnov said he might have some clues as to why this is happening. 
diff --git a/caracal/dispatch_crew/caltables.py b/caracal/dispatch_crew/caltables.py
index 255f1dfc..bebf7b00 100644
--- a/caracal/dispatch_crew/caltables.py
+++ b/caracal/dispatch_crew/caltables.py
@@ -23,7 +23,7 @@ def calibrator_database():
 
     # There isn't a Southern standard in CASA
     # so construct a little database of them for reference
-    caracal.log.info("Obtaining divine knowledge from: %s" % __DB_FILENAME)
+    caracal.log.info("Obtaining divine knowledge from %s" % __DB_FILENAME)
 
     __CALIBRATOR_DB = cp.catalog_parser(__DB_FILENAME)
     #caracal.log.info("\n" + str(__CALIBRATOR_DB))
@@ -36,6 +36,6 @@ def casa_calibrator_database():
     global __CASA_CALIBRATOR_DB
     if __CASA_CALIBRATOR_DB is not None:
         return __CASA_CALIBRATOR_DB
-    caracal.log.info("Obtaining divine knowledge from: %s" % __DB_CASA_FILENAME)
+    caracal.log.info("Obtaining divine knowledge from %s" % __DB_CASA_FILENAME)
     __CASA_CALIBRATOR_DB = cp.catalog_parser(__DB_CASA_FILENAME)
     return __CASA_CALIBRATOR_DB
diff --git a/caracal/dispatch_crew/config_parser.py b/caracal/dispatch_crew/config_parser.py
index 09623f64..38ae2200 100644
--- a/caracal/dispatch_crew/config_parser.py
+++ b/caracal/dispatch_crew/config_parser.py
@@ -272,7 +272,7 @@ class config_parser(object):
             dtype = None
 
             def typecast(val):
-                """Helper function to cast value to excpected type.
+                """Helper function to cast value to expected type.
                 If string=True, bools are cast to string bools, so value is suitable for command-line parsing"""
                 if is_list and isinstance(val, list):
                     return [typecast(x) for x in val]
@@ -321,21 +321,21 @@ class config_parser(object):
             if options is not None:
                 if hasattr(options, attr_name):
                     optval = getattr(options, attr_name)
-                    # parse lists
-                    if is_list:
-                        optval = str2list(optval)
+                    # optval is always a string, so...
+                    # ...parse lists or dicts as yaml objects
+                    if is_list or dtype is dict:
+                        optval = yaml.safe_load(optval)
+                    # ...and typecast to expected type
                     option_value = typecast(optval)
                     if option_value != default_value:
                         caracal.log.info("  command line sets --{} = {}".format(option_name, option_value))
                     groups[key] = option_value
             # else populate parser with default value
             else:
-                # lists correspond to multiple arguments
-                if is_list:
-                    # no support for lists of dicts from the command line, for now
-                    if dtype is not dict:
-                        self._parser.add_argument("--" + option_name, help=argparse.SUPPRESS, type=str,
-                                                  default=",".join(map(value2str, default_value)))
+                # lists and dicts expressed via yaml
+                if is_list or dtype is dict:
+                    self._parser.add_argument("--" + option_name, help=argparse.SUPPRESS, type=str,
+                                                default=yaml.safe_dump(default_value))
                 # booleans have a choice
                 elif dtype is bool:
                     self._parser.add_argument("--" + option_name, help=argparse.SUPPRESS,
diff --git a/caracal/main.py b/caracal/main.py
index 73b39d9c..7d1e9120 100644
--- a/caracal/main.py
+++ b/caracal/main.py
@@ -168,9 +168,6 @@ def execute_pipeline(options, config, block):
 #        with stream_director(log) as director:  # stdout and stderr needs to go to the log as well -- nah
 
         try:
-            # Obtain some divine knowledge
-            cdb = mkct.calibrator_database()
-
             pipeline = worker_administrator(config,
                            workers_directory,
                            add_all_first=False, prefix=options.general_prefix,
@@ -181,6 +178,9 @@ def execute_pipeline(options, config, block):
             if options.report:
                 pipeline.regenerate_reports()
             else:
+                ## OMS: I don't think this is necessary, as it is not used here directly, and loaded on-demand
+                # # Obtain some divine knowledge
+                # cdb = mkct.calibrator_database()
                 pipeline.run_workers()
         except SystemExit as e:
             # if e.code != 0:
@@ -227,6 +227,8 @@ def main(argv):
             parser.error("unknown worker '{}'".format(options.worker_help))
         return
 
+    caracal.log.info(f"Invoked as {' '.join(sys.argv)}")
+
     # User requests default config => dump and exit
     if options.get_default:
         sample_config = SAMPLE_CONFIGS.get(options.get_default_template)
diff --git a/caracal/schema/__init__.py b/caracal/schema/__init__.py
index 798f2fe8..89fdfd56 100644
--- a/caracal/schema/__init__.py
+++ b/caracal/schema/__init__.py
@@ -1,2 +1,2 @@
-SCHEMA_VERSION = "1.1.0"
+SCHEMA_VERSION = "1.0.4"
 
diff --git a/caracal/schema/crosscal_schema.yml b/caracal/schema/crosscal_schema.yml
index e034f117..0a1037b8 100644
--- a/caracal/schema/crosscal_schema.yml
+++ b/caracal/schema/crosscal_schema.yml
@@ -212,7 +212,7 @@ mapping:
             seq:
               - type: str
             required: false
-            example: "120s, inf, None 120s"
+            example: "120s, inf, None, 120s"
           combine:
             desc: For each step in 'order' above, set along what axis the data should be combined before solving. Options are '' (i.e., no data combination; solutions break at obs, scan, field, and spw boundarie), 'obs', 'scan', 'spw', 'field'. To combine along multiple axes use comma-separated axis names in a single string, e.g., 'obs,scan'. This setting is only relevant for the steps of type K, G and B included in 'order' above. For A steps this setting is ignored and an empty string may be used.
             seq:
diff --git a/caracal/workers/crosscal_worker.py b/caracal/workers/crosscal_worker.py
index 0e5680b9..fb548d6a 100644
--- a/caracal/workers/crosscal_worker.py
+++ b/caracal/workers/crosscal_worker.py
@@ -19,6 +19,19 @@ from casacore.tables import table
 NAME = "Cross-calibration"
 LABEL = 'crosscal'
 
+def check_config(config, name):
+    for primsec in "primary", "secondary":
+        order = config[primsec]["order"]
+        # check that all order steps are legal
+        invalid = [x for x in order if x not in RULES]
+        if invalid:
+            raise caracal.ConfigurationError(f"{name}: {primsec}: order: invalid steps {','.join(invalid)}")
+        # check that numbers match
+        for other in "calmode", "solint", "combine":
+            if len(config[primsec][other]) != len(order):
+                raise caracal.ConfigurationError(f"{name}: {primsec}: {other}: expected {len(order)} elements, found {len(config[primsec][other])}")
+
+
 # E.g. to split out continuum/<dir> from output/continuum/dir
 def get_dir_path(string, pipeline):
     return string.split(pipeline.output)[1][1:]
diff --git a/caracal/workers/line_worker.py b/caracal/workers/line_worker.py
index be2a18c2..882e4385 100644
--- a/caracal/workers/line_worker.py
+++ b/caracal/workers/line_worker.py
@@ -7,22 +7,23 @@ import stimela.dismissable as sdm
 import stimela.recipe
 import astropy
 import shutil
+import itertools
 from astropy.io import fits
-import caracal
 # Modules useful to calculate common barycentric frequency grid
 from astropy.time import Time
 from astropy.coordinates import SkyCoord
 from astropy.coordinates import EarthLocation
 from astropy import constants
+import psutil
 import astropy.units as units
 import re
 import datetime
 import numpy as np
-import yaml
+import caracal
 from caracal.dispatch_crew import utils
-import itertools
 from caracal.workers.utils import manage_flagsets as manflags
-import psutil
+from caracal import log
+from caracal.workers.utils import remove_output_products
 
 NAME = 'Process and Image Line Data'
 LABEL = 'line'
@@ -439,6 +440,10 @@ def worker(pipeline, recipe, config):
                        label='{0:s}:: Add model column'.format(step))
 
         msname_mst = add_ms_label(msname, "mst")
+        msname_mst_base = os.path.splitext(msname_mst)[0]
+        flagv = msname_mst + ".flagversions"
+        summary_file = f'{msname_mst_base}-summary.json'
+        obsinfo_file = f'{msname_mst_base}-obsinfo.txt'
 
         if pipeline.enable_task(config, 'mstransform'):
             # Set UVLIN fit channel range
@@ -506,10 +511,7 @@ def worker(pipeline, recipe, config):
                 fitspw = config['mstransform']['uvlin']['fitspw']
 
             # If the output of this run of mstransform exists, delete it first
-            if os.path.exists('{0:s}/{1:s}'.format(pipeline.msdir, msname_mst)) or \
-                    os.path.exists('{0:s}/{1:s}.flagversions'.format(pipeline.msdir, msname_mst)):
-                os.system(
-                    'rm -rf {0:s}/{1:s} {0:s}/{1:s}.flagversions'.format(pipeline.msdir, msname_mst))
+            remove_output_products((msname_mst, flagv, summary_file, obsinfo_file), directory=pipeline.msdir, log=log)
 
             col = config['mstransform']['col']
             step = 'mstransform-ms{:d}'.format(i)
@@ -535,7 +537,6 @@ def worker(pipeline, recipe, config):
                        output=pipeline.output,
                        label='{0:s}:: Doppler tracking corrections'.format(step))
 
-            msname_mst_base = os.path.splitext(msname_mst)[0]
             if config['mstransform']['obsinfo']:
                 step = 'listobs-ms{:d}'.format(i)
                 recipe.add('cab/casa_listobs',
diff --git a/caracal/workers/obsconf_worker.py b/caracal/workers/obsconf_worker.py
index 358aac35..2f0b433a 100644
--- a/caracal/workers/obsconf_worker.py
+++ b/caracal/workers/obsconf_worker.py
@@ -197,36 +197,40 @@ def worker(pipeline, recipe, config):
         # The order of fields here is important
         for term in "target gcal fcal bpcal xcal".split():
             conf_fields = getattr(pipeline, term)[i]
+            conf_fields_str = ','.join(conf_fields)
             label, fields = intents[term]
-            label = ",".join(label)
-            # check if user set fields manually
-            if set(all_fields).intersection(conf_fields):
-                label = term
-                if term == 'target':
-                    pipeline.target[i] = [value for value in getattr(pipeline, term)[i] if value in all_fields]
-            elif fields in [None, []]:
-                getattr(pipeline, term)[i] = []
-                continue
-            elif "all" in conf_fields:
-                getattr(pipeline, term)[i] = fields
-            elif "longest" in conf_fields:
+            label = ",".join(label) or term
+            #  no fields set for this term -- make empty and continue
+            if not conf_fields:
+                found_fields = []
+                # check if user set fields manually
+            elif conf_fields_str == "all":
+                found_fields = fields
+            elif conf_fields_str == "longest":
                 f = utils.observed_longest(msdict, fields)
-                getattr(pipeline, term)[i] = [f]
-            elif "nearest" in conf_fields:
+                found_fields = [f] if f else []
+            elif conf_fields_str == "nearest":
                 f = utils.select_gcal(msdict, fields, mode="nearest")
-                getattr(pipeline, term)[i] = [f]
+                found_fields = [f] if f else []
             else:
-                raise RuntimeError("Could not find field/selection {0}."\
-                        " Please check the [observation_config.{1}] "\
-                        "section of the config file".format(conf_fields, term))
+                found_fields = set(all_fields).intersection(conf_fields)
+
+            getattr(pipeline, term)[i] = list(found_fields)
+            if not found_fields:
+            # complain if not found, unless it's 'xcal', which is only for polcal, so let that worker complain
+                if term != "xcal":
+                    raise RuntimeError(f"Can't find an appropriate FIELD for obsinfo: {term}: {conf_fields_str}. "
+                                        "Please check this config setting. It may also be that your MS scan intents "
+                                        f"are not pupulated correctly, in which case you must set {term} to a list of explicit field names.")
+                continue
 
 #            caracal.log.info("    ====================================")
-            caracal.log.info(f"  {label}:")
+            caracal.log.info(f"  {term} ({label}):")
 #            caracal.log.info("     ---------------------------------- ")
             _ra = []
             _dec = []
             _fid = []
-            for f in getattr(pipeline, term)[i]:
+            for f in found_fields:
                 fid = utils.get_field_id(msdict, f)[0]
                 targetpos = targetinfo['REFERENCE_DIR'][fid][0]
                 ra = targetpos[0]/np.pi*180
diff --git a/caracal/workers/selfcal_worker.py b/caracal/workers/selfcal_worker.py
index c87537cd..eede055c 100644
--- a/caracal/workers/selfcal_worker.py
+++ b/caracal/workers/selfcal_worker.py
@@ -50,7 +50,7 @@ SOL_TERMS_INDEX = {
     "DD": 2,
 }
 
-def check_config(config):
+def check_config(config, name):
     """
     Optional function to check consistency of config, invoked before the pipeline runs.
     its purpose is to log warnings, or raise exceptions on bad errors.
@@ -58,10 +58,10 @@ def check_config(config):
     # First let' check that we are not using transfer gains with meqtrees or not starting at the start with meqtrees
     if config['calibrate_with'].lower() == 'meqtrees':
         if config['transfer_apply_gains']['enable']:
-            raise caracal.UserInputError(
+            raise caracal.ConfigurationError(
                 'Gains cannot be interpolated with MeqTrees, please switch to CubiCal. Exiting.')
         if int(config['start_iter']) != 1:
-            raise caracal.UserInputError(
+            raise caracal.ConfigurationError(
                 "We cannot reapply MeqTrees calibration at a given step. Hence you will need to do a full selfcal loop.")
         if int(config['cal_cubical']['chan_chunk']) != -1:
             caracal.log.info("The channel chunk has no effect on MeqTrees.")
@@ -69,19 +69,19 @@ def check_config(config):
             caracal.log.info("Delay selfcal does not work with MeqTrees, please switch to Cubical. Exiting.")
     else:
         if int(config['start_iter']) != 1:
-            raise caracal.UserInputError(
+            raise caracal.ConfigurationError(
                 "We cannot reapply Cubical calibration at a given step. Hence you will need to do a full selfcal loop.")
     # First check we are actually running a calibrate
     if config['calibrate']['enable']:
         # Running with a model shorter than the output type is dengerous with 'CORR_RES'
         if 'CORR_RES' in  config['calibrate']['output_data']:
             if len(config['calibrate']['model']) < config['cal_niter']:
-                raise caracal.UserInputError(
+                raise caracal.ConfigurationError(
                     "You did not set a model to use for every iteration while using residuals. This is too dangerous for CARACal to execute.")
 
         # Make sure we are not using two_step with CubiCal
         if config['calibrate_with'].lower() == 'cubical' and config['cal_meqtrees']['two_step']:
-            raise caracal.UserInputError(
+            raise caracal.ConfigurationError(
                 "Two_Step calibration is an experimental mode only available for meqtrees at the moment.")
         #Then let's check that the solutions are reasonable and fit in our time chunks
         #!!!!!! Remainder solutions are not checked to be a full solution block!!!!!!!!
@@ -128,14 +128,14 @@ def check_config(config):
                 caracal.log.error("You are using all timeslots in your solutions (i.e. 0) but have set cal_timeslots_chunk, please set it to 0 for using all timeslots.")
                 caracal.log.error("Your timeslots chunk = {}".format(time_chunk))
                 caracal.log.error("Your timeslots solutions to be applied are {}".format(', '.join([str(x) for x in solutions])))
-                raise caracal.UserInputError("Inconsistent selfcal chunking")
+                raise caracal.ConfigurationError("Inconsistent selfcal chunking")
             sol_int_array = float(time_chunk)/np.array(solutions,dtype=float)
             for val in sol_int_array:
                 if val != int(val):
                     caracal.log.error("Not all applied time solutions fit in the timeslot_chunk.")
                     caracal.log.error("Your timeslot chunk = {}".format(time_chunk))
                     caracal.log.error("Your time solutions to be applied are {}".format(', '.join([str(x) for x in solutions])))
-                    raise caracal.UserInputError("Inconsistent selfcal chunking")
+                    raise caracal.ConfigurationError("Inconsistent selfcal chunking")
         # Then we repeat for the channels, as these arrays do not have to be the same length as the timeslots this can not be combined
         # This is not an option for meqtrees
         if config['calibrate_with'].lower() == 'cubical':
@@ -177,14 +177,14 @@ def check_config(config):
                     caracal.log.error("You are using all channels in your solutions (i.e. 0) but have set chan_chunk, please set it to 0 for using all channels.")
                     caracal.log.error("Your channel chunk = {} \n".format(chan_chunk))
                     caracal.log.error("Your channel solutions to be applied are {}".format(', '.join([str(x) for x in solutions])))
-                    raise caracal.UserInputError("Inconsistent selfcal chunking")
+                    raise caracal.ConfigurationError("Inconsistent selfcal chunking")
                 sol_int_array = float(chan_chunk)/np.array(solutions,dtype=float)
                 for val in sol_int_array:
                     if val != int(val):
                         caracal.log.error("Not all applied channel solutions fit in the chan_chunk.")
                         caracal.log.error("Your channel chunk = {} \n".format(chan_chunk))
                         caracal.log.error("Your channel solutions to be applied are {}".format(', '.join([str(x) for x in solutions])))
-                        raise caracal.UserInputError("Inconsistent selfcal chunking")
+                        raise caracal.ConfigurationError("Inconsistent selfcal chunking")
     # Check some imaging stuff
     if config['image']['enable']:
         if config['img_maxuv_l'] > 0. and  config['img_taper'] > 0.:
diff --git a/caracal/workers/transform_worker.py b/caracal/workers/transform_worker.py
index 23e16fde..6677f677 100644
--- a/caracal/workers/transform_worker.py
+++ b/caracal/workers/transform_worker.py
@@ -3,14 +3,12 @@ import os
 import sys
 import caracal
 import stimela.dismissable as sdm
-import getpass
 import stimela.recipe
-import re
 import json
-import numpy as np
-from caracal.dispatch_crew import utils
-from caracal.workers.utils import manage_fields as manfields
+import caracal
 from caracal.workers.utils import manage_flagsets as manflags
+from caracal import log
+from caracal.workers.utils import remove_output_products
 
 NAME = 'Transform Data by Splitting/Average/Applying calibration'
 LABEL = 'transform'
@@ -39,10 +37,10 @@ table_suffix = {
 _target_fields = {'target'}
 _cal_fields = set("fcal bpcal gcal xcal".split())
 
-def get_fields_to_split(config):
+def get_fields_to_split(config, name):
     fields = config['field']
     if not fields:
-        raise ValueError("split_field: field cannot be empty")
+        raise caracal.ConfigurationError(f"'{name}: field' cannot be empty")
     elif fields == 'calibrators':
         return _cal_fields
     elif fields == 'target':
@@ -52,15 +50,15 @@ def get_fields_to_split(config):
         fields_to_split = set(fields.split(','))
         diff = fields_to_split.difference(_cal_fields)
         if diff:
-            raise ValueError(
-                "split_field: field: expected 'target', 'calibrators', or one or more of {}. Got '{}'".format(
+            raise caracal.ConfigurationError(
+                "'{}: field: expected 'target', 'calibrators', or one or more of {}. Got '{}'".format(name,
                     ', '.join([f"'{f}'" for f in _cal_fields]), ','.join(diff)
                 ))
         return fields_to_split
 
 
-def check_config(config):
-    get_fields_to_split(config)
+def check_config(config, name):
+    get_fields_to_split(config, name)
 
 
 def worker(pipeline, recipe, config):
@@ -70,7 +68,7 @@ def worker(pipeline, recipe, config):
     label_in = config['label_in']
     label_out = config['label_out']
     from_target = True if label_in and config['field'] == 'target' else False
-    field_to_split = get_fields_to_split(config)
+    field_to_split = get_fields_to_split(config, wname)
     # are we splitting calibrators
     splitting_cals = field_to_split.intersection(_cal_fields)
     if (pipeline.enable_task(config, 'split_field') or pipeline.enable_task(config, 'changecentre')) and pipeline.enable_task(config, 'concat'):
@@ -178,14 +176,20 @@ def worker(pipeline, recipe, config):
                         config, flags_before_worker, flags_after_worker)
 
             flagv = to_ms + '.flagversions'
+            if pipeline.enable_task(config, 'split_field'):
+                msbase = os.path.splitext(to_ms)[0]
+                obsinfo_msname = to_ms
+            else:
+                msbase = pipeline.msbasenames[i]
+                obsinfo_msname = from_ms
+
+            summary_file = f'{msbase}-summary.json'
+            obsinfo_file = f'{msbase}-obsinfo.txt'
 
             if pipeline.enable_task(config, 'split_field'):
-                step = 'split_field-ms{0:d}-{1:d}'.format(i,target_iter)
+                step = 'split_field-ms{0:d}-{1:d}'.format(i, target_iter)
                 # If the output of this run of mstransform exists, delete it first
-                if os.path.exists('{0:s}/{1:s}'.format(pipeline.msdir, to_ms)) or \
-                        os.path.exists('{0:s}/{1:s}'.format(pipeline.msdir, flagv)):
-                    os.system(
-                        'rm -rf {0:s}/{1:s} {0:s}/{2:s}'.format(pipeline.msdir, to_ms, flagv))
+                remove_output_products((to_ms, flagv, summary_file, obsinfo_file), directory=pipeline.msdir, log=log)
 
                 recipe.add('cab/casa_mstransform', step,
                            {
@@ -278,6 +282,7 @@ def worker(pipeline, recipe, config):
 
             if pipeline.enable_task(config, 'obsinfo'):
                 if (config['obsinfo']['listobs']):
+
                     if pipeline.enable_task(config, 'split_field') or transform_mode == 'concat':
                         listfile = '{0:s}-obsinfo.txt'.format(os.path.splitext(to_ms)[0])
                     else:
@@ -287,7 +292,7 @@ def worker(pipeline, recipe, config):
                     recipe.add('cab/casa_listobs', step,
                                {
                                    "vis": obsinfo_msname,
-                                   "listfile": listfile+":msfile",
+                                   "listfile": obsinfo_file+":msfile",
                                    "overwrite": True,
                                },
                                input=pipeline.input,
@@ -295,6 +300,7 @@ def worker(pipeline, recipe, config):
                                label='{0:s}:: Get observation information ms={1:s}'.format(step, obsinfo_msname))
 
                 if (config['obsinfo']['summary_json']):
+
                     if pipeline.enable_task(config, 'split_field') or transform_mode == 'concat':
                         listfile = '{0:s}-summary.json'.format(os.path.splitext(to_ms)[0])
                     else:
@@ -306,7 +312,7 @@ def worker(pipeline, recipe, config):
                                    "msname": obsinfo_msname,
                                    "command": 'summary',
                                    "display": False,
-                                   "outfile": listfile+":msfile"
+                                   "outfile": summary_file+":msfile"
                                },
                                input=pipeline.input,
                                output=pipeline.obsinfo,
diff --git a/caracal/workers/utils/__init__.py b/caracal/workers/utils/__init__.py
index e69de29b..f54e7670 100644
--- a/caracal/workers/utils/__init__.py
+++ b/caracal/workers/utils/__init__.py
@@ -0,0 +1,13 @@
+import os
+
+def remove_output_products(files, directory=None, log=None):
+    """
+    Removes output products (given by a list of files), in a directory (if specified)
+    """
+    for fullpath in files:
+        if directory:
+            fullpath = os.path.join(directory, fullpath)
+        if os.path.exists(fullpath):
+            if log is not None:
+                log.info(f'removing pre-existing {fullpath}')
+            os.system(f'rm -rf {fullpath}')
diff --git a/caracal/workers/worker_administrator.py b/caracal/workers/worker_administrator.py
index 968345a6..a50811a5 100644
--- a/caracal/workers/worker_administrator.py
+++ b/caracal/workers/worker_administrator.py
@@ -32,20 +32,23 @@ class worker_administrator(object):
         self.config = config
         self.singularity_image_dir = singularity_image_dir
         self.container_tech = container_tech
+        for key in "msdir input output".split():
+            if not self.config['general'].get(key):
+                raise caracal.ConfigurationError(f"'general: {key}' must be specified")
+
         self.msdir = self.config['general']['msdir']
         self.input = self.config['general']['input']
         self.output = self.config['general']['output']
-        self.obsinfo = self.config['general']['output'] + '/obsinfo'
-        self.reports = self.config['general']['output'] + '/reports'
-        self.diagnostic_plots = self.config['general']['output'] + \
-            '/diagnostic_plots'
-        self.configFolder = self.config['general']['output'] + '/cfgFiles'
-        self.caltables = self.config['general']['output'] + '/caltables'
-        self.masking = self.config['general']['output'] + '/masking'
-        self.continuum = self.config['general']['output'] + '/continuum'
-        self.crosscal_continuum = self.config['general']['output'] + '/continuum/crosscal'
-        self.cubes = self.config['general']['output'] + '/cubes'
-        self.mosaics = self.config['general']['output'] + '/mosaics'
+        self.obsinfo = f'{self.output}/obsinfo'
+        self.reports = f'{self.output}/reports'
+        self.diagnostic_plots = f'{self.output}/diagnostic_plots'
+        self.configFolder = f'{self.output}/cfgFiles'
+        self.caltables = f'{self.output}/caltables'
+        self.masking = f'{self.output}/masking'
+        self.continuum = f'{self.output}/continuum'
+        self.crosscal_continuum = f'{self.output}/continuum/crosscal'
+        self.cubes = f'{self.output}/cubes'
+        self.mosaics = f'{self.output}/mosaics'
         self.generate_reports = generate_reports
         self.timeNow = '{:%Y%m%d-%H%M%S}'.format(datetime.now())
         self.ms_extension = self.config["getdata"]["extension"]
@@ -53,13 +56,13 @@ class worker_administrator(object):
 
         self._msinfo_cache = {}
 
-        self.logs_symlink = self.config['general']['output'] + '/logs'
+        self.logs_symlink = f'{self.output}/logs'
         self.logs = "{}-{}".format(self.logs_symlink, self.timeNow)
 
-        if not self.config['general']['rawdatadir']:
+
+        self.rawdatadir = self.config['general']['rawdatadir']
+        if not self.rawdatadir:
             self.rawdatadir = self.config['general']['rawdatadir'] = self.msdir
-        else:
-            self.rawdatadir = self.config['general']['rawdatadir']
 
         self.virtconcat = False
         self.workers_directory = workers_directory
@@ -121,24 +124,8 @@ class worker_administrator(object):
         self.recipes = {}
         # Workers to skip
         self.skip = []
-        # Initialize empty lists for ddids, leave this up to get data worker to define
-        self.init_names([])
-        self.init_pipeline(prep_input=config["general"]["prep_workspace"])
-
-        # save configuration file
-        configFileName = os.path.splitext(configFileName)[0]
-        outConfigName = '{0:s}_{1:s}.yml'.format(
-            configFileName, self.timeNow)
-
-        with open(os.path.join(self.configFolder, outConfigName), 'w') as outfile:
-            ruamel.yaml.dump(self.config, outfile, Dumper=ruamel.yaml.RoundTripDumper)
-
-    def init_names(self, dataid):
-        """ iniitalize names to be used throughout the pipeline and associated
-            general fields that must be propagated
-        """
-        self.dataid = dataid
-
+        # Initialize empty lists for ddids, leave this up to getdata worker to define
+        self.dataid = []
         # names of all MSs
         self.msnames = []
         # basenames of all MSs (sans extension)
@@ -146,26 +133,55 @@ class worker_administrator(object):
         # filename prefixes for outputs (formed up as prefix-msbase)
         self.prefix_msbases = []
 
-        for item in 'rawdatadir input msdir output'.split():
-            value = getattr(self, item, None)
-            if value:
-                setattr(self, item, value)
+        ## OMS skipping this here, leave it to the getdata
+        # self.init_names([], allow_empty=True)
+        self.init_pipeline(prep_input=config["general"]["prep_workspace"])
+
+        # save configuration files
+
+        config_base = os.path.splitext(configFileName)[0]
+        outConfigOrigName = f'{self.configFolder}/{config_base}-{self.timeNow}.orig.yml'
+        outConfigName = f'{self.configFolder}/{config_base}-{self.timeNow}.yml'
+
+        log.info(f"Saving original configuration file as {outConfigOrigName}")
+        shutil.copyfile(configFileName, outConfigOrigName)  # original config
 
-        if self.rawdatadir:
-            for dataid in self.dataid:
-                pattern = os.path.join(self.rawdatadir, f"{dataid}.{self.ms_extension}")
-                msnames = [os.path.basename(ms) for ms in glob.glob(pattern)]
-                if len(msnames) == 0 and not self.ignore_missing:
-                    raise RuntimeError(f"The pattern/path '{pattern}' did not "\
-                          f"return files. Please double check your"\
-                          " general: msdir, general: rawdatadir, and/or getdata: dataid "\
-                          "settings in your config file. However, if you wish to proceed regardless,"\
-                          " you can set getdata: ignore_missing: true")
-                msbases = [os.path.splitext(ms)[0] for ms in msnames]
-                self.msnames += msnames
-                self.msbasenames += msbases
-                self.prefix_msbases += [ f"{self.prefix}-{x}" for x in msbases]
-            self.nobs = len(self.msnames)
+        log.info(f"Saving full configuration as {outConfigName}")
+        with open(outConfigName, 'w') as outfile:           # config+command line
+            ruamel.yaml.dump(self.config, outfile, Dumper=ruamel.yaml.RoundTripDumper)
+
+    def init_names(self, dataids):
+        """ iniitalize names to be used throughout the pipeline and associated
+            general fields that must be propagated
+        """
+        ## OMS: this was a very elaborate no-op, away with it
+        # for item in 'rawdatadir input msdir output'.split():
+        #     value = getattr(self, item, None)
+        #     if value:
+        #         setattr(self, item, value)
+
+        self.dataid = list(filter(bool, dataids))
+        if not self.dataid:
+            raise caracal.ConfigurationError(f"Empty 'getdata: dataid' entry")
+        patterns =  [f"{dataid}.{self.ms_extension}" for dataid in self.dataid]
+
+        for pattern in patterns:
+            msnames = [os.path.basename(ms) for ms in glob.glob(os.path.join(self.rawdatadir, pattern))]
+            if not msnames:
+                if self.ignore_missing:
+                    log.warning(f"'{pattern}' did not match any files, but getdata: ignore_missing is set, proceeding anyway")
+                else:
+                    raise caracal.ConfigurationError(f"'{pattern}' did not match any files under {self.rawdatadir}. Check your "
+                            "'general: msdir/rawdatadir' and/or 'getdata: dataid/extension' settings, or "
+                            "set 'getdata: ignore_missing: true'")
+            msbases = [os.path.splitext(ms)[0] for ms in msnames]
+            self.msnames += msnames
+            self.msbasenames += msbases
+            self.prefix_msbases += [ f"{self.prefix}-{x}" for x in msbases]
+        self.nobs = len(self.msnames)
+        if not self.nobs:
+            raise caracal.ConfigurationError(f"No matching input data found in {self.rawdatadir} for {','.join(patterns)}. Check your "
+                " 'general: msdir/rawdatadir' and/or 'getdata: dataid/extension' settings.")
 
         for item in 'refant fcal bpcal gcal target xcal'.split():
             value = getattr(self, item, None)
@@ -274,7 +290,7 @@ class worker_administrator(object):
         if not os.path.exists(self.output):
             os.mkdir(self.output)
         if not os.path.exists(self.rawdatadir):
-            os.mkdir(self.rawdatadir)
+            raise caracal.ConfigurationError(f"{self.rawdatadir} does not exist, check your general config section")
         if not os.path.exists(self.obsinfo):
             os.mkdir(self.obsinfo)
         if not os.path.exists(self.logs):
@@ -360,7 +376,7 @@ class worker_administrator(object):
                 log.error('Error importing worker "{0:s}" from {1:s}'.format(_worker, self.workers_directory))
                 raise
             if hasattr(worker, 'check_config'):
-                worker.check_config(config)
+                worker.check_config(config, name=_name)
             # check for cab specs
             cabspecs = cabspecs_general
             if config["cabs"]:
['caracal/main.py', 'caracal/workers/utils/__init__.py', 'caracal/dispatch_crew/config_parser.py', 'caracal/workers/crosscal_worker.py', 'caracal/workers/line_worker.py', 'caracal/workers/transform_worker.py', 'caracal/dispatch_crew/caltables.py', 'caracal/schema/__init__.py', 'caracal/workers/obsconf_worker.py', 'caracal/workers/worker_administrator.py', 'caracal/workers/selfcal_worker.py', 'caracal/schema/crosscal_schema.yml']