flake8 complaints from PR
PR #751 is generating some interesting complaints, and some of them appear to be real bugs:

Running flake8 on changed files

- [ ] algorithms/refinement/parameterisation/configure.py:639:39: F821 undefined name 'beam'
- [ ] algorithms/refinement/parameterisation/configure.py:656:31: F821 undefined name 'beam'
- [x] algorithms/refinement/refiner.py:727:20: F841 local variable 'num_rows' is assigned to but never used
- [x] algorithms/refinement/refiner.py:735:13: F402 import 'flex' from line 23 shadowed by loop variable
- [x] algorithms/refinement/refiner.py:796:12: E713 test for membership should be 'not in'
- [x] algorithms/refinement/refiner.py:872:17: F841 local variable 'temp' is assigned to but never used
- [x] algorithms/refinement/refiner.py:926:13: F841 local variable 'temp' is assigned to but never used
- [x] algorithms/refinement/reflection_manager.py:180:19: F821 undefined name 'Sorry'
- [x] algorithms/refinement/reflection_manager.py:281:23: F821 undefined name 'Sorry'
- [x] algorithms/refinement/reflection_manager.py:293:23: F821 undefined name 'Sorry'
- [x] command_line/generate_tutorial_text.py:344:8: E713 test for membership should be 'not in'
- [x] command_line/generate_tutorial_text.py:347:8: E713 test for membership should be 'not in'
- [x] test/algorithms/refinement/test_constraints.py:18:1: F401 'libtbx.test_utils.open_tmp_directory' imported but unused
- [x] test/algorithms/refinement/test_constraints.py:171:5: F841 local variable 'result' is assigned to but never used
- [x] test/algorithms/refinement/test_refine_multi_wedges.py:13:1: F401 'dials.array_family.flex' imported but unused
- [x] test/command_line/test_refine.py:40:5: F841 local variable 'result' is assigned to but never used
- [x] test/command_line/test_refine.py:94:5: F841 local variable 'result1' is assigned to but never used
- [x] test/command_line/test_refine.py:95:5: F841 local variable 'result2' is assigned to but never used
- [x] test/command_line/test_refine.py:144:5: F841 local variable 'result1' is assigned to but never used
- [x] test/command_line/test_refine.py:177:5: F401 'random.randint' imported but unused
- [x] test/command_line/test_refine.py:177:5: F401 'random.uniform' imported but unused

Working through these now
diff --git a/algorithms/refinement/parameterisation/configure.py b/algorithms/refinement/parameterisation/configure.py
index 95c2df4eb..36b00c930 100644
--- a/algorithms/refinement/parameterisation/configure.py
+++ b/algorithms/refinement/parameterisation/configure.py
@@ -75,8 +75,9 @@
     }
 
     scan_varying = False
-      .help = "Allow models that are not forced to be static to vary during the"
-              "scan"
+      .help = "Allow models that are not forced to be static to vary during"
+              "the scan, Auto will run one macrocycle with static then"
+              "scan varying refinement for the crystal"
       .type = bool
       .short_caption = "Scan-varying refinement"
 
diff --git a/algorithms/refinement/refiner.py b/algorithms/refinement/refiner.py
index 770d06345..dadb345d7 100644
--- a/algorithms/refinement/refiner.py
+++ b/algorithms/refinement/refiner.py
@@ -236,6 +236,15 @@ def _build_components(cls, params, reflections, experiments, verbosity):
             raise Sorry("Cannot refine a mixture of stills and scans")
         do_stills = exps_are_stills[0]
 
+        # If experiments are stills, ensure scan-varying refinement won't be attempted
+        if do_stills:
+            params.refinement.parameterisation.scan_varying = False
+
+        # Refiner does not accept scan_varying=Auto. This is a special case for
+        # doing macrocycles of refinement in dials.refine.
+        if params.refinement.parameterisation.scan_varying is libtbx.Auto:
+            params.refinement.parameterisation.scan_varying = False
+
         # calculate reflection block_width if required for scan-varying refinement
         if params.refinement.parameterisation.scan_varying:
             from dials.algorithms.refinement.reflection_manager import BlockCalculator
@@ -656,6 +665,9 @@ def __init__(
 
         self._verbosity = verbosity
 
+        # Keep track of whether this is stills or scans type refinement
+        self.experiment_type = refman.experiment_type
+
         return
 
     def get_experiments(self):
@@ -712,7 +724,7 @@ def get_parameter_correlation_matrix(self, step, col_select=None):
             col_select = range(len(all_labels))
         sel = string_sel(col_select, all_labels)
         labels = [e for e, s in zip(all_labels, sel) if s]
-        num_cols = num_rows = len(labels)
+        num_cols = len(labels)
         if num_cols == 0:
             return None, None
 
@@ -720,8 +732,6 @@ def get_parameter_correlation_matrix(self, step, col_select=None):
 
             assert corrmat.is_square_matrix()
 
-            from scitbx.array_family import flex
-
             idx = flex.bool(sel).iselection()
             sub_corrmat = flex.double(flex.grid(num_cols, num_cols))
 
@@ -781,7 +791,7 @@ def print_out_of_sample_rmsd_table(self):
         rad2deg = 180 / pi
 
         # check if it makes sense to proceed
-        if not "out_of_sample_rmsd" in self._refinery.history:
+        if "out_of_sample_rmsd" not in self._refinery.history:
             return
         nref = len(self.get_free_reflections())
         if nref < 10:
@@ -857,7 +867,6 @@ def print_exp_rmsd_table(self):
 
             scan = exp.scan
             try:
-                temp = scan.get_oscillation(deg=False)
                 images_per_rad = 1.0 / abs(scan.get_oscillation(deg=False)[1])
             except (AttributeError, ZeroDivisionError):
                 images_per_rad = None
@@ -911,7 +920,6 @@ def print_panel_rmsd_table(self):
             )
         scan = self._experiments.scans()[0]
         try:
-            temp = scan.get_oscillation(deg=False)
             images_per_rad = 1.0 / abs(scan.get_oscillation(deg=False)[1])
         except AttributeError:
             images_per_rad = None
diff --git a/algorithms/refinement/reflection_manager.py b/algorithms/refinement/reflection_manager.py
index 7e7b9ee66..e8c7856c8 100644
--- a/algorithms/refinement/reflection_manager.py
+++ b/algorithms/refinement/reflection_manager.py
@@ -177,6 +177,8 @@ def _check_scan_range(exp_phi, scan):
         # scan edges
         start, stop = scan.get_oscillation_range(deg=False)
         if min(exp_phi) - start > 0.087266 or stop - max(exp_phi) > 0.087266:
+            from dials.util import Sorry
+
             raise Sorry("The reflections do not fill the scan range.")
 
     def per_width(self, width, deg=True):
@@ -278,6 +280,8 @@ def from_parameters_reflections_experiments(
             refman = StillsReflectionManager
             # check incompatible weighting strategy
             if params.weighting_strategy.override == "statistical":
+                from dials.util import Sorry
+
                 raise Sorry(
                     'The "statistical" weighting strategy is not compatible '
                     "with stills refinement"
@@ -290,6 +294,8 @@ def from_parameters_reflections_experiments(
                     'The "{0}" weighting strategy is not compatible with '
                     "scan refinement"
                 ).format(params.weighting_strategy.override)
+                from dials.util import Sorry
+
                 raise Sorry(msg)
 
         # set automatic outlier rejection options
@@ -384,6 +390,7 @@ class ReflectionManager(object):
     sampling to form the working subset."""
 
     _weighting_strategy = weighting_strategies.StatisticalWeightingStrategy()
+    experiment_type = "scans"
 
     def __init__(
         self,
@@ -897,6 +904,7 @@ class StillsReflectionManager(ReflectionManager):
     about X, Y, DelPsi residuals"""
 
     _weighting_strategy = weighting_strategies.StillsWeightingStrategy()
+    experiment_type = "stills"
 
     def _id_refs_to_keep(self, obs_data):
         """Create a selection of observations that pass certain conditions.
diff --git a/command_line/generate_tutorial_text.py b/command_line/generate_tutorial_text.py
index d9f82d07d..cd8502536 100644
--- a/command_line/generate_tutorial_text.py
+++ b/command_line/generate_tutorial_text.py
@@ -101,7 +101,7 @@ class dials_reindex(Job):
         cmd = "dials.reindex indexed.pickle change_of_basis_op=a,b,c"
 
     class dials_refine(Job):
-        cmd = "dials.refine bravais_setting_9.json reindexed_reflections.pickle"
+        cmd = "dials.refine bravais_setting_9.json reindexed_reflections.pickle scan_varying=false"
 
     class dials_sv_refine(Job):
         cmd = "dials.refine refined_experiments.json refined.pickle scan_varying=true"
@@ -236,7 +236,7 @@ def generate_processing_detail_text_betalactamase():
         ("dials.reindex", "dials.reindex indexed.pickle change_of_basis_op=a+b,-a+b,c"),
         (
             "dials.refine",
-            "dials.refine bravais_setting_2.json reindexed_reflections.pickle",
+            "dials.refine bravais_setting_2.json reindexed_reflections.pickle scan_varying=false",
         ),
         (
             "dials.sv_refine",
@@ -341,9 +341,9 @@ def extract_last_indexed_spot_count(path):
         sys.exit(0)
 
     # As a quick development hack, add option for only the newer process
-    if not "--beta" in sys.argv:
+    if "--beta" not in sys.argv:
         print("Generating thaumatin tutorial")
         generate_processing_detail_text_thaumatin()
-    if not "--thaum" in sys.argv:
+    if "--thaum" not in sys.argv:
         print("Generating betalactamase tutorial")
         generate_processing_detail_text_betalactamase()
diff --git a/command_line/refine.py b/command_line/refine.py
index a788e1fb8..2d3f8d688 100644
--- a/command_line/refine.py
+++ b/command_line/refine.py
@@ -11,18 +11,8 @@
 #  included in the root directory of this package.
 
 # DIALS_ENABLE_COMMAND_LINE_COMPLETION
-from __future__ import absolute_import, division, print_function
-
-import logging
-from dials.util import Sorry
-from libtbx import Auto
-from dials.array_family import flex
-from dials.algorithms.refinement import RefinerFactory
-
-logger = logging.getLogger("dials.command_line.refine")
-
-help_message = """
 
+"""
 Refine the diffraction geometry of input experiments against the input indexed
 reflections. For rotation scans, the model may be either static (the same for
 all reflections) or scan-varying (dependent on image number in the scan).
@@ -34,14 +24,24 @@
 
   dials.refine experiments.json indexed.pickle
 
-  dials.refine experiments.json indexed.pickle scan_varying=True
+  dials.refine experiments.json indexed.pickle scan_varying=(False/True/Auto)
 
 """
 
-# The phil scope
-from libtbx.phil import parse
+from __future__ import absolute_import, division, print_function
+import sys
+import logging
+from time import time
+import dials.util
+import libtbx.phil
+from libtbx import Auto
+from dials.array_family import flex
+from dials.algorithms.refinement import RefinerFactory
+
+logger = logging.getLogger("dials.command_line.refine")
 
-phil_scope = parse(
+# The phil scope
+phil_scope = libtbx.phil.parse(
     """
 
   output {
@@ -116,11 +116,9 @@
       .expert_level = 1
   }
 
-  do_scan_varying_macrocycle = False
-    .type = bool
-    .help = "Override whatever scan_varying is set to and force one round of"
-            "static refinement followed by one round of scan-varying refinement"
-    .expert_level = 3
+  n_static_macrocycles = 1
+    .type = int(value_min=1)
+    .help = "Number of macro-cycles of static refinement to perform"
 
   include scope dials.algorithms.refinement.refiner.phil_scope
 """,
@@ -128,399 +126,442 @@
 )
 
 # local overrides for refiner.phil_scope
-phil_overrides = parse(
+phil_overrides = libtbx.phil.parse(
     """
   refinement
   {
     verbosity = 2
+    parameterisation.scan_varying = Auto
   }
 """
 )
 
 working_phil = phil_scope.fetch(sources=[phil_overrides])
 
+def write_centroids_table(refiner, filename):
+    """Write a table of centroids from refinement for debugging.
 
-class Script(object):
-    """A class for running the script."""
+    Args:
+        refiner: A dials.algorithms.refinement.refiner.Refiner.
+        filename (str): The name of the file to which to write.
 
-    def __init__(self, phil=working_phil):
-        """Initialise the script."""
-        from dials.util.options import OptionParser
-        import libtbx.load_env
+    Returns:
+        None
+    """
 
-        # The script usage
-        usage = (
-            "usage: %s [options] [param.phil] "
-            "experiments.json reflections.pickle" % libtbx.env.dispatcher_name
-        )
+    matches = refiner.get_matches()
+
+    header = (
+        "H\tK\tL\tFrame_obs\tX_obs\tY_obs\tPhi_obs\tX_calc\t" "Y_calc\tPhi_calc"
+    )
+    msg_temp = "%d\t%d\t%d\t%d\t%5.3f\t%5.3f\t%9.6f\t%5.3f\t%5.3f\t%9.6f"
+    has_del_psi = "delpsical.rad" in matches
+    if has_del_psi:
+        header += "\tDelta_Psi"
+        msg_temp += "\t%9.6f"
+    header += "\n"
+    msg_temp += "\n"
+
+    with open(filename, "w") as f:
+        f.write(header)
+
+        for m in matches:
+            (h, k, l) = m["miller_index"]
+            frame = m["xyzobs.px.value"][2]
+            x_obs, y_obs, phi_obs = m["xyzobs.mm.value"]
+            x_calc, y_calc, phi_calc = m["xyzcal.mm"]
+            if has_del_psi:
+                del_psi = m["delpsical.rad"]
+                msg = msg_temp % (
+                    h,
+                    k,
+                    l,
+                    frame,
+                    x_obs,
+                    y_obs,
+                    phi_obs,
+                    x_calc,
+                    y_calc,
+                    phi_calc,
+                    del_psi,
+                )
+            else:
+                msg = msg_temp % (
+                    h,
+                    k,
+                    l,
+                    frame,
+                    x_obs,
+                    y_obs,
+                    phi_obs,
+                    x_calc,
+                    y_calc,
+                    phi_calc,
+                )
+            f.write(msg)
 
-        # Create the parser
-        self.parser = OptionParser(
-            usage=usage,
-            phil=phil,
-            read_reflections=True,
-            read_experiments=True,
-            check_format=False,
-            epilog=help_message,
-        )
+def run_macrocycle(params, reflections, experiments):
+    """Run one macrocycle of refinement.
 
-    def write_centroids_table(self, refiner, filename):
+    One macrocycle of refinement is run, as specified by the PHIL
+    parameters, using the centroids from the supplied reflections
+    and the initial experimental geometry taken from experiments.
 
-        matches = refiner.get_matches()
 
-        header = (
-            "H\tK\tL\tFrame_obs\tX_obs\tY_obs\tPhi_obs\tX_calc\t" "Y_calc\tPhi_calc"
-        )
-        msg_temp = "%d\t%d\t%d\t%d\t%5.3f\t%5.3f\t%9.6f\t%5.3f\t%5.3f\t%9.6f"
-        has_del_psi = "delpsical.rad" in matches
-        if has_del_psi:
-            header += "\tDelta_Psi"
-            msg_temp += "\t%9.6f"
-        header += "\n"
-        msg_temp += "\n"
-
-        with open(filename, "w") as f:
-            f.write(header)
-
-            for m in matches:
-                (h, k, l) = m["miller_index"]
-                frame = m["xyzobs.px.value"][2]
-                x_obs, y_obs, phi_obs = m["xyzobs.mm.value"]
-                x_calc, y_calc, phi_calc = m["xyzcal.mm"]
-                if has_del_psi:
-                    del_psi = m["delpsical.rad"]
-                    msg = msg_temp % (
-                        h,
-                        k,
-                        l,
-                        frame,
-                        x_obs,
-                        y_obs,
-                        phi_obs,
-                        x_calc,
-                        y_calc,
-                        phi_calc,
-                        del_psi,
-                    )
-                else:
-                    msg = msg_temp % (
-                        h,
-                        k,
-                        l,
-                        frame,
-                        x_obs,
-                        y_obs,
-                        phi_obs,
-                        x_calc,
-                        y_calc,
-                        phi_calc,
-                    )
-                f.write(msg)
+    Args:
+        params: The working PHIL parameters.
+        reflections: A reflection table containing observed centroids
+        experiments: The initial dxtbx experimental geometry models
 
-    @staticmethod
-    def check_input(reflections):
-        """Check the input is suitable for refinement. So far just check keys in
-        the reflection table. Maybe later check experiments have overlapping models
-        etc."""
+    Returns:
+        tuple: The Refiner, the reflection table with updated predictions
+            and flags, and the refinement history object.
+    """
+    # Get the refiner
+    logger.info("Configuring refiner")
+    refiner = RefinerFactory.from_parameters_data_experiments(
+        params, reflections, experiments
+    )
+
+    # Refine the geometry
+    nexp = len(experiments)
+    if nexp == 1:
+        logger.info("Performing refinement of a single Experiment...")
+    else:
+        logger.info("Performing refinement of {0} Experiments...".format(nexp))
+
+    # Refine and get the refinement history
+    history = refiner.run()
+
+    # Update predictions for all indexed reflections
+    logger.info("Updating predictions for indexed reflections")
+    preds = refiner.predict_for_indexed()
+
+    # just copy over the columns of interest or columns that may have been
+    # updated, leaving behind things added by e.g. scan-varying refinement
+    # such as 'block', the U, B and UB matrices and gradients.
+    for key in preds.keys():
+        if key in reflections.keys() or key in [
+            "s1",
+            "xyzcal.mm",
+            "xyzcal.px",
+            "entering",
+            "delpsical.rad",
+        ]:
+            reflections[key] = preds[key]
+
+    # set refinement flags
+    assert len(preds) == len(reflections)
+    reflections.unset_flags(
+        flex.size_t_range(len(reflections)),
+        reflections.flags.excluded_for_refinement
+        | reflections.flags.used_in_refinement
+        | reflections.flags.centroid_outlier
+        | reflections.flags.predicted,
+    )
+    reflections.set_flags(
+        preds.get_flags(preds.flags.excluded_for_refinement),
+        reflections.flags.excluded_for_refinement,
+    )
+    reflections.set_flags(
+        preds.get_flags(preds.flags.centroid_outlier),
+        reflections.flags.centroid_outlier,
+    )
+    reflections.set_flags(
+        preds.get_flags(preds.flags.used_in_refinement),
+        reflections.flags.used_in_refinement,
+    )
+    reflections.set_flags(
+        preds.get_flags(preds.flags.predicted), reflections.flags.predicted
+    )
+
+    return refiner, reflections, history
+
+def run_dials_refine(experiments, reflections, params):
+    """Functional interface to tasks performed by the program dials.refine.
+
+    This runs refinement according to the PHIL parameters in params, using
+    taking the initial experimental geometry from experiments, and the
+    observed reflection centroids from reflections.
+
+    Static refinement can be done in a number of macrocycles. By default,
+    for scans, this will be followed by a macrocycle of scan-varying
+    refinement.
+
+
+    Args:
+        experiments: The initial dxtbx experimental geometry models
+        reflections: A reflection table containing observed centroids
+        params: The working PHIL parameters
+
+    Returns:
+        tuple: The refined experiments, the updated reflection table, the
+            Refiner object and the refinement history object.
+    """
 
-        msg = (
-            "The supplied reflection table does not have the required data "
-            + "column: {0}"
-        )
-        for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
-            if key not in reflections:
-                msg = msg.format(key)
-                raise Sorry(msg)
+    # Modify options if necessary
+    if params.output.correlation_plot.filename is not None:
+        params.refinement.refinery.journal.track_parameter_correlation = True
+
+    # scan_varying=Auto is special to dials.refine. Refiner expects it to be
+    # True or False, so catch and set here
+    scan_varying = params.refinement.parameterisation.scan_varying
+    if scan_varying is Auto:
+        params.refinement.parameterisation.scan_varying = False
+
+    if params.n_static_macrocycles == 1:
+        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
+        experiments = refiner.get_experiments()
+    else:
+        for i in range(params.n_static_macrocycles):
+            logger.info("\nStatic refinement macrocycle {0}".format(i + 1))
+            refiner, reflections, history = run_macrocycle(params, reflections, experiments)
+            experiments = refiner.get_experiments()
+
+    # Scan-varying macrocycle, if appropriate
+    if scan_varying is Auto and refiner.experiment_type == 'scans':
+        logger.info("\nScan-varying refinement")
+        params.refinement.parameterisation.scan_varying = True
+        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
+        experiments = refiner.get_experiments()
+
+    return experiments, reflections, refiner, history
 
-        # FIXME add other things to be checked here
-        return
+def run(args=None, phil=working_phil):
+    """
+    Set up refinement from command line options, files and PHIL parameters.
+    Run refinement and save output files as specified.
+
+    Called when running dials.refine as a command-line program
 
-    @staticmethod
-    def run_macrocycle(params, reflections, experiments):
+    Args:
+        args (list): Additional command-line arguments
+        phil: The working PHIL parameters
 
-        # Get the refiner
-        logger.info("Configuring refiner")
-        refiner = RefinerFactory.from_parameters_data_experiments(
-            params, reflections, experiments
+    Returns:
+        None
+    """
+
+    import dials.util.log
+    from dials.util.options import OptionParser
+    from dials.util.options import flatten_reflections
+    from dials.util.options import flatten_experiments
+    import libtbx.load_env
+    import six.moves.cPickle as pickle
+
+    start_time = time()
+
+    # The script usage
+    usage = (
+        "usage: %s [options] [param.phil] "
+        "experiments.json reflections.pickle" % libtbx.env.dispatcher_name
+    )
+
+    # Create the parser
+    parser = OptionParser(
+        usage=usage,
+        phil=phil,
+        read_reflections=True,
+        read_experiments=True,
+        check_format=False,
+        epilog=__doc__)
+
+    # Parse the command line
+    params, options = parser.parse_args(args=args, show_diff_phil=False)
+    reflections = flatten_reflections(params.input.reflections)
+    experiments = flatten_experiments(params.input.experiments)
+
+    # Configure the logging
+    dials.util.log.config(
+        info=params.output.log,
+        debug=params.output.debug_log,
+    )
+
+    # Try to load the models and data
+    nexp = len(experiments)
+    if nexp == 0:
+        sys.exit("No Experiments found in the input")
+    if len(reflections) == 0:
+        sys.exit("No reflection data found in the input")
+    if len(reflections) > 1:
+        sys.exit("Only one reflections list can be imported at present")
+    reflections = reflections[0]
+
+    # check input is suitable
+    msg = (
+        "The supplied reflection table does not have the required data "
+        + "column: {0}"
+    )
+    for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
+        if key not in reflections:
+            msg = msg.format(key)
+            raise dials.util.Sorry(msg)
+
+    from dials.util.version import dials_version
+    logger.info(dials_version())
+
+    # Log the diff phil
+    diff_phil = parser.diff_phil.as_str()
+    if diff_phil:
+        logger.info("The following parameters have been modified:\n")
+        logger.info(diff_phil)
+
+    # Warn about potentially unhelpful options
+    if params.refinement.mp.nproc > 1:
+        logger.warning(
+            "WARNING: setting nproc > 1 is only helpful in rare "
+            "circumstances. It is not recommended for typical data processing "
+            "tasks.\n"
         )
 
-        # Refine the geometry
-        nexp = len(experiments)
-        if nexp == 1:
-            logger.info("Performing refinement of a single Experiment...")
-        else:
-            logger.info("Performing refinement of {0} Experiments...".format(nexp))
-
-        # Refine and get the refinement history
-        history = refiner.run()
-        return refiner, history
-
-    def run(self, args=None):
-        """Execute the script."""
-        from time import time
-        import six.moves.cPickle as pickle
-        from dials.util import log
-        from dials.util.options import flatten_reflections, flatten_experiments
-
-        start_time = time()
-
-        # Parse the command line
-        params, options = self.parser.parse_args(args=args, show_diff_phil=False)
-        reflections = flatten_reflections(params.input.reflections)
-        experiments = flatten_experiments(params.input.experiments)
-
-        # Try to load the models and data
-        nexp = len(experiments)
-        if nexp == 0:
-            print("No Experiments found in the input")
-            self.parser.print_help()
-            return
-        if len(reflections) == 0:
-            print("No reflection data found in the input")
-            self.parser.print_help()
-            return
-        if len(reflections) > 1:
-            raise Sorry("Only one reflections list can be imported at present")
-        reflections = reflections[0]
-
-        self.check_input(reflections)
-
-        if __name__ == "__main__":
-            # Configure the logging
-            log.config(info=params.output.log, debug=params.output.debug_log)
-
-        from dials.util.version import dials_version
-
-        logger.info(dials_version())
-
-        # Log the diff phil
-        diff_phil = self.parser.diff_phil.as_str()
-        if diff_phil is not "":
-            logger.info("The following parameters have been modified:\n")
-            logger.info(diff_phil)
-
-        # Warn about potentially unhelpful options
-        if params.refinement.mp.nproc > 1:
-            logger.warning(
-                "WARNING: setting nproc > 1 is only helpful in rare "
-                "circumstances. It is not recommended for typical data processing "
-                "tasks.\n"
-            )
+    # Run refinement
+    experiments, reflections, refiner, history = run_dials_refine(experiments, reflections, params)
 
-        # Modify options if necessary
-        if params.output.correlation_plot.filename is not None:
-            params.refinement.refinery.journal.track_parameter_correlation = True
-        do_sv_macrocycle = False
-        if params.do_scan_varying_macrocycle:
-            params.refinement.parameterisation.scan_varying = False
-            do_sv_macrocycle = True
-
-        # Do refinement
-        if do_sv_macrocycle:
-            logger.info("\nScan-static refinement")
-            refiner, history = self.run_macrocycle(params, reflections, experiments)
-            logger.info("\nScan-varying refinement")
-            params.refinement.parameterisation.scan_varying = True
-            refiner, history = self.run_macrocycle(params, reflections, experiments)
-        else:
-            refiner, history = self.run_macrocycle(params, reflections, experiments)
+    # For the usual case of refinement of one crystal, print that model for information
+    crystals = experiments.crystals()
+    if len(crystals) == 1:
+        logger.info("")
+        logger.info("Final refined crystal model:")
+        logger.info(crystals[0])
+
+    # Write table of centroids to file, if requested
+    if params.output.centroids:
+        logger.info(
+            "Writing table of centroids to '{0}'".format(params.output.centroids)
+        )
+        write_centroids_table(refiner, params.output.centroids)
 
-        if params.output.centroids:
+    # Write scan-varying parameters to file, if there were any
+    if params.output.parameter_table:
+        scans = experiments.scans()
+        if len(scans) > 1:
             logger.info(
-                "Writing table of centroids to '{0}'".format(params.output.centroids)
+                "Writing a scan-varying parameter table is only supported "
+                "for refinement of a single scan"
             )
-            self.write_centroids_table(refiner, params.output.centroids)
-
-        # Get the refined experiments
-        experiments = refiner.get_experiments()
-
-        # Write scan-varying parameters to file, if there were any
-        if params.output.parameter_table:
-            scans = experiments.scans()
-            if len(scans) > 1:
+        else:
+            scan = scans[0]
+            text = refiner.get_param_reporter().varying_params_vs_image_number(
+                scan.get_array_range()
+            )
+            if text:
                 logger.info(
-                    "Writing a scan-varying parameter table is only supported "
-                    "for refinement of a single scan"
+                    "Writing scan-varying parameter table to {0}".format(
+                        params.output.parameter_table
+                    )
                 )
+                f = open(params.output.parameter_table, "w")
+                f.write(text)
+                f.close()
             else:
-                scan = scans[0]
-                text = refiner.get_param_reporter().varying_params_vs_image_number(
-                    scan.get_array_range()
-                )
-                if text:
-                    logger.info(
-                        "Writing scan-varying parameter table to {0}".format(
-                            params.output.parameter_table
-                        )
-                    )
-                    f = open(params.output.parameter_table, "w")
-                    f.write(text)
-                    f.close()
-                else:
-                    logger.info("No scan-varying parameter table to write")
-
-        crystals = experiments.crystals()
-        if len(crystals) == 1:
-            # output the refined model for information
-            logger.info("")
-            logger.info("Final refined crystal model:")
-            logger.info(crystals[0])
-
-        # Save the refined experiments to file
-        output_experiments_filename = params.output.experiments
+                logger.info("No scan-varying parameter table to write")
+
+    # Save the refined experiments to file
+    output_experiments_filename = params.output.experiments
+    logger.info(
+        "Saving refined experiments to {0}".format(output_experiments_filename)
+    )
+    from dxtbx.model.experiment_list import ExperimentListDumper
+
+    dump = ExperimentListDumper(experiments)
+    dump.as_json(output_experiments_filename)
+
+    # Save reflections with updated predictions if requested (allow to switch
+    # this off if it is a time-consuming step)
+    if params.output.reflections:
         logger.info(
-            "Saving refined experiments to {0}".format(output_experiments_filename)
-        )
-        from dxtbx.model.experiment_list import ExperimentListDumper
-
-        dump = ExperimentListDumper(experiments)
-        dump.as_json(output_experiments_filename)
-
-        # Save reflections with updated predictions if requested (allow to switch
-        # this off if it is a time-consuming step)
-        if params.output.reflections:
-            # Update predictions for all indexed reflections
-            logger.info("Updating predictions for indexed reflections")
-            preds = refiner.predict_for_indexed()
-
-            # just copy over the columns of interest or columns that may have been
-            # updated, leaving behind things added by e.g. scan-varying refinement
-            # such as 'block', the U, B and UB matrices and gradients.
-            for key in preds.keys():
-                if key in reflections.keys() or key in [
-                    "s1",
-                    "xyzcal.mm",
-                    "xyzcal.px",
-                    "entering",
-                    "delpsical.rad",
-                ]:
-                    reflections[key] = preds[key]
-
-            # set refinement flags
-            assert len(preds) == len(reflections)
-            reflections.unset_flags(
-                flex.size_t_range(len(reflections)),
-                reflections.flags.excluded_for_refinement
-                | reflections.flags.used_in_refinement
-                | reflections.flags.centroid_outlier
-                | reflections.flags.predicted,
-            )
-            reflections.set_flags(
-                preds.get_flags(preds.flags.excluded_for_refinement),
-                reflections.flags.excluded_for_refinement,
-            )
-            reflections.set_flags(
-                preds.get_flags(preds.flags.centroid_outlier),
-                reflections.flags.centroid_outlier,
-            )
-            reflections.set_flags(
-                preds.get_flags(preds.flags.used_in_refinement),
-                reflections.flags.used_in_refinement,
-            )
-            reflections.set_flags(
-                preds.get_flags(preds.flags.predicted), reflections.flags.predicted
+            "Saving reflections with updated predictions to {0}".format(
+                params.output.reflections
             )
+        )
+        if params.output.include_unused_reflections:
+            reflections.as_pickle(params.output.reflections)
+        else:
+            sel = reflections.get_flags(reflections.flags.used_in_refinement)
+            reflections.select(sel).as_pickle(params.output.reflections)
 
-            logger.info(
-                "Saving reflections with updated predictions to {0}".format(
-                    params.output.reflections
-                )
+    # Save matches to file for debugging
+    if params.output.matches:
+        matches = refiner.get_matches()
+        logger.info(
+            "Saving matches (use for debugging purposes) to {0}".format(
+                params.output.matches
             )
-            if params.output.include_unused_reflections:
-                reflections.as_pickle(params.output.reflections)
-            else:
-                sel = reflections.get_flags(reflections.flags.used_in_refinement)
-                reflections.select(sel).as_pickle(params.output.reflections)
+        )
+        matches.as_pickle(params.output.matches)
 
-        # For debugging, if requested save matches to file
-        if params.output.matches:
-            matches = refiner.get_matches()
-            logger.info(
-                "Saving matches (use for debugging purposes) to {0}".format(
-                    params.output.matches
-                )
-            )
-            matches.as_pickle(params.output.matches)
+    # Create correlation plots
+    if params.output.correlation_plot.filename is not None:
+        from os.path import splitext
 
-        # Correlation plot
-        if params.output.correlation_plot.filename is not None:
-            from os.path import splitext
+        root, ext = splitext(params.output.correlation_plot.filename)
+        if not ext:
+            ext = ".pdf"
 
-            root, ext = splitext(params.output.correlation_plot.filename)
-            if not ext:
-                ext = ".pdf"
+        steps = params.output.correlation_plot.steps
+        if steps is None:
+            steps = [history.get_nrows() - 1]
 
-            steps = params.output.correlation_plot.steps
-            if steps is None:
-                steps = [history.get_nrows() - 1]
+        # extract individual column names or indices
+        col_select = params.output.correlation_plot.col_select
 
-            # extract individual column names or indices
-            col_select = params.output.correlation_plot.col_select
+        num_plots = 0
+        for step in steps:
+            fname_base = root
+            if len(steps) > 1:
+                fname_base += "_step%02d" % step
 
-            num_plots = 0
-            for step in steps:
-                fname_base = root
-                if len(steps) > 1:
-                    fname_base += "_step%02d" % step
+            corrmats, labels = refiner.get_parameter_correlation_matrix(
+                step, col_select
+            )
+            if [corrmats, labels].count(None) == 0:
+                from dials.algorithms.refinement.refinement_helpers import corrgram
 
-                corrmats, labels = refiner.get_parameter_correlation_matrix(
-                    step, col_select
-                )
-                if [corrmats, labels].count(None) == 0:
-                    from dials.algorithms.refinement.refinement_helpers import corrgram
-
-                    for resid_name, corrmat in corrmats.items():
-                        plot_fname = fname_base + "_" + resid_name + ext
-                        plt = corrgram(corrmat, labels)
-                        if plt is not None:
-                            logger.info(
-                                "Saving parameter correlation plot to {}".format(
-                                    plot_fname
-                                )
-                            )
-                            plt.savefig(plot_fname)
-                            plt.close()
-                            num_plots += 1
-                    mat_fname = fname_base + ".pickle"
-                    with open(mat_fname, "wb") as handle:
-                        for k, corrmat in corrmats.items():
-                            corrmats[k] = corrmat.as_scitbx_matrix()
+                for resid_name, corrmat in corrmats.items():
+                    plot_fname = fname_base + "_" + resid_name + ext
+                    plt = corrgram(corrmat, labels)
+                    if plt is not None:
                         logger.info(
-                            "Saving parameter correlation matrices to {0}".format(
-                                mat_fname
+                            "Saving parameter correlation plot to {}".format(
+                                plot_fname
                             )
                         )
-                        pickle.dump({"corrmats": corrmats, "labels": labels}, handle)
-
-            if num_plots == 0:
-                msg = (
-                    "Sorry, no parameter correlation plots were produced. Please set "
-                    "track_parameter_correlation=True to ensure correlations are "
-                    "tracked, and make sure correlation_plot.col_select is valid."
-                )
-                logger.info(msg)
-
-        # Write out refinement history, if requested
-        if params.output.history:
-            with open(params.output.history, "wb") as handle:
-                logger.info(
-                    "Saving refinement step history to {0}".format(
-                        params.output.history
+                        plt.savefig(plot_fname)
+                        plt.close()
+                        num_plots += 1
+                mat_fname = fname_base + ".pickle"
+                with open(mat_fname, "wb") as handle:
+                    for k, corrmat in corrmats.items():
+                        corrmats[k] = corrmat.as_scitbx_matrix()
+                    logger.info(
+                        "Saving parameter correlation matrices to {0}".format(
+                            mat_fname
+                        )
                     )
-                )
-                pickle.dump(history, handle)
+                    pickle.dump({"corrmats": corrmats, "labels": labels}, handle)
 
-        # Log the total time taken
-        logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
+        if num_plots == 0:
+            msg = (
+                "Sorry, no parameter correlation plots were produced. Please set "
+                "track_parameter_correlation=True to ensure correlations are "
+                "tracked, and make sure correlation_plot.col_select is valid."
+            )
+            logger.info(msg)
 
-        return experiments, reflections
+    # Save refinement history
+    if params.output.history:
+        with open(params.output.history, "wb") as handle:
+            logger.info(
+                "Saving refinement step history to {0}".format(
+                    params.output.history
+                )
+            )
+            pickle.dump(history, handle)
 
+    # Log the total time taken
+    logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
 
-if __name__ == "__main__":
-    from dials.util import halraiser
+if __name__ == '__main__':
+  with dials.util.show_mail_on_error():
+    run()
 
-    try:
-        script = Script()
-        script.run()
-    except Exception as e:
-        halraiser(e)
diff --git a/doc/sphinx/documentation/tutorials/dials_for_ed.rst b/doc/sphinx/documentation/tutorials/dials_for_ed.rst
index ca9da138a..b426c6fbc 100644
--- a/doc/sphinx/documentation/tutorials/dials_for_ed.rst
+++ b/doc/sphinx/documentation/tutorials/dials_for_ed.rst
@@ -448,7 +448,7 @@ Indexing
 Refinement of the experimental geometry was stabilised by fixing the detector
 distance, and :math:`\tau_2` and :math:`\tau_3` rotations. To do this, a PHIL
 parameter file was created in each processing directory for use in indexing and
-refinement steps.
+static refinement steps.
 
 .. code-block:: bash
 
@@ -458,6 +458,7 @@ refinement steps.
       detector {
         fix_list = "Dist,Tau2,Tau3"
       }
+      scan_varying=false
     }
   }
   EOF
diff --git a/doc/sphinx/documentation/tutorials/multi_crystal_analysis.rst b/doc/sphinx/documentation/tutorials/multi_crystal_analysis.rst
index aaa287934..e54e26d5c 100644
--- a/doc/sphinx/documentation/tutorials/multi_crystal_analysis.rst
+++ b/doc/sphinx/documentation/tutorials/multi_crystal_analysis.rst
@@ -143,7 +143,7 @@ script does. If time is *really* short then try uncommenting the line
         return
 
       # static model refinement
-      cmd = "dials.refine experiments.json indexed.pickle " + \
+      cmd = "dials.refine experiments.json indexed.pickle scan_varying=false " + \
             "outlier.algorithm=tukey use_all_reflections=true"
       easy_run.fully_buffered(command=cmd)
       if not os.path.isfile("refined_experiments.json"):
@@ -263,7 +263,7 @@ no manual intervention.
 
 Following indexing we do scan-static cell refinement::
 
-  dials.refine experiments.json indexed.pickle outlier.algorithm=tukey use_all_reflections=true
+  dials.refine experiments.json indexed.pickle scan_varying=false outlier.algorithm=tukey use_all_reflections=true
 
 Outlier rejection was switched on in an attempt to avoid any zingers or other
 errant spots from affecting our refined cells. Without analysing the data closer
@@ -507,7 +507,7 @@ between the detector or beam parameters with individual crystals. As motivation
 we may look at these correlations for one of these datasets. For example::
 
   cd sweep_00
-  dials.refine experiments.json indexed.pickle \
+  dials.refine experiments.json indexed.pickle scan_varying=false \
     track_parameter_correlation=true correlation_plot.filename=corrplot.png
   cd ..
 
@@ -690,7 +690,7 @@ crystal refinement job. First we try outlier rejection, so that the refinement
 run is similar to the jobs we ran on individual datasets::
 
   dials.refine combined_experiments.json combined_reflections.pickle \
-    use_all_reflections=true outlier.algorithm=tukey
+    scan_varying=false use_all_reflections=true outlier.algorithm=tukey
 
 ::
 
@@ -759,7 +759,7 @@ because it selectively removes reflections from the worst fitting experiments.
 Instead we try without outlier rejection::
 
   dials.refine combined_experiments.json combined_reflections.pickle \
-    use_all_reflections=true \
+    scan_varying=false use_all_reflections=true \
     output.experiments=refined_combined_experiments.json
 
 This worked much better::
@@ -875,6 +875,7 @@ perhaps we should use outlier rejection though. Now the models are close enough
 it is safe to do so::
 
   dials.refine refined_combined_experiments.json combined_reflections.pickle \
+    scan_varying=false \
     use_all_reflections=true \
     outlier.algorithm=tukey \
     output.experiments=refined_combined_experiments_outrej.json
diff --git a/doc/sphinx/documentation/tutorials/small_molecule_tutorial.rst b/doc/sphinx/documentation/tutorials/small_molecule_tutorial.rst
index 3f14f21ed..f0ce34903 100644
--- a/doc/sphinx/documentation/tutorials/small_molecule_tutorial.rst
+++ b/doc/sphinx/documentation/tutorials/small_molecule_tutorial.rst
@@ -81,7 +81,7 @@ Prior to integration we want to refine the experimental geometry and the scan va
 
 .. code-block:: bash
 
-   dials.refine indexed.pickle experiments.json output.reflections=static.pickle output.experiments=static.json
+   dials.refine indexed.pickle experiments.json output.reflections=static.pickle output.experiments=static.json scan_varying=false
    dials.refine static.pickle static.json scan_varying=True
 
 At this stage the reciprocal lattice view will show a much improved level of agreement between the indexed reflections from the four sweeps:
diff --git a/test/algorithms/refinement/test_constraints.py b/test/algorithms/refinement/test_constraints.py
index 25e58939b..689cb88f4 100644
--- a/test/algorithms/refinement/test_constraints.py
+++ b/test/algorithms/refinement/test_constraints.py
@@ -15,7 +15,7 @@
 from __future__ import absolute_import, division, print_function
 import os
 from copy import deepcopy
-from libtbx.test_utils import open_tmp_directory, approx_equal
+from libtbx.test_utils import approx_equal
 from libtbx import easy_run
 from scitbx import sparse
 from dials.array_family import flex
@@ -166,9 +166,9 @@ def test_constrained_refinement(dials_regression, run_in_tmpdir):
     cmd = (
         "dials.refine foo_experiments.json foo_reflections.pickle "
         "history=history.pickle refinement.parameterisation.detector."
-        "constraints.parameter=Dist"
+        "constraints.parameter=Dist scan_varying=False"
     )
-    result = easy_run.fully_buffered(command=cmd).raise_if_errors()
+    easy_run.fully_buffered(command=cmd).raise_if_errors()
     # load refinement history
     import six.moves.cPickle as pickle
 
diff --git a/test/algorithms/refinement/test_refine_multi_wedges.py b/test/algorithms/refinement/test_refine_multi_wedges.py
index 404655e2a..08f6e83b6 100644
--- a/test/algorithms/refinement/test_refine_multi_wedges.py
+++ b/test/algorithms/refinement/test_refine_multi_wedges.py
@@ -10,7 +10,7 @@
 from scitbx import matrix
 from dxtbx.model.experiment_list import ExperimentListFactory
 from six.moves import cPickle as pickle
-from dials.array_family import flex
+from dials.array_family import flex  # noqa: F401
 import pytest
 
 
@@ -51,6 +51,7 @@ def test(dials_regression, run_in_tmpdir):
             "dials.refine",
             "combined_experiments.json",
             "combined_reflections.pickle",
+            "scan_varying=false",
             "outlier.algorithm=null",
             "close_to_spindle_cutoff=0.05",
         ]
@@ -111,6 +112,7 @@ def test_order_invariance(dials_regression, run_in_tmpdir):
             "dials.refine",
             "combined_experiments.json",
             "combined_reflections.pickle",
+            "scan_varying=false",
             "outlier.algorithm=tukey",
             "history=history1.pickle",
             "output.experiments=refined_experiments1.json",
@@ -144,6 +146,7 @@ def test_order_invariance(dials_regression, run_in_tmpdir):
             "dials.refine",
             "combined_experiments.json",
             "combined_reflections.pickle",
+            "scan_varying=false",
             "outlier.algorithm=tukey",
             "history=history2.pickle",
             "output.experiments=refined_experiments2.json",
diff --git a/test/command_line/test_refine.py b/test/command_line/test_refine.py
index 2a4c1a2ec..103561025 100644
--- a/test/command_line/test_refine.py
+++ b/test/command_line/test_refine.py
@@ -31,13 +31,13 @@ def test1(dials_regression, run_in_tmpdir):
     # set some old defaults
     cmd = (
         "dials.refine close_to_spindle_cutoff=0.05 reflections_per_degree=100 "
-        + "outlier.separate_blocks=False "
+        + "outlier.separate_blocks=False scan_varying=False "
         + experiments_path
         + " "
         + pickle_path
     )
 
-    result = easy_run.fully_buffered(command=cmd).raise_if_errors()
+    easy_run.fully_buffered(command=cmd).raise_if_errors()
     # load results
     reg_exp = ExperimentListFactory.from_json_file(
         os.path.join(data_dir, "regression_experiments.json"), check_format=False
@@ -77,7 +77,7 @@ def test2(dials_regression, run_in_tmpdir):
         + experiments_path
         + " "
         + pickle_path
-        + " reflections_per_degree=50 "
+        + " scan_varying=False reflections_per_degree=50 "
         " outlier.algorithm=null close_to_spindle_cutoff=0.05"
     )
     cmd2 = (
@@ -91,8 +91,8 @@ def test2(dials_regression, run_in_tmpdir):
         " set_scan_varying_errors=True"
     )
 
-    result1 = easy_run.fully_buffered(command=cmd1).raise_if_errors()
-    result2 = easy_run.fully_buffered(command=cmd2).raise_if_errors()
+    easy_run.fully_buffered(command=cmd1).raise_if_errors()
+    easy_run.fully_buffered(command=cmd2).raise_if_errors()
 
     # load and check results
     with open("history.pickle", "rb") as fh:
@@ -141,7 +141,7 @@ def test3(dials_regression, run_in_tmpdir):
         "crystal.orientation.smoother.interval_width_degrees=auto "
         "crystal.unit_cell.smoother.interval_width_degrees=auto"
     )
-    result1 = easy_run.fully_buffered(command=cmd1).raise_if_errors()
+    easy_run.fully_buffered(command=cmd1).raise_if_errors()
 
     # load and check results
     with open("history.pickle", "rb") as fh:
@@ -174,7 +174,6 @@ def test4():
     from dials_refinement_helpers_ext import surpl_iter as surpl
 
     # Borrowed from tst_reflection_table function tst_find_overlapping
-    from random import randint, uniform
 
     N = 110
     r = flex.reflection_table.empty_standard(N)
['command_line/generate_tutorial_text.py', 'doc/sphinx/documentation/tutorials/dials_for_ed.rst', 'algorithms/refinement/parameterisation/configure.py', 'command_line/refine.py', 'test/command_line/test_refine.py', 'algorithms/refinement/reflection_manager.py', 'algorithms/refinement/refiner.py', 'doc/sphinx/documentation/tutorials/small_molecule_tutorial.rst', 'test/algorithms/refinement/test_refine_multi_wedges.py', 'test/algorithms/refinement/test_constraints.py', 'doc/sphinx/documentation/tutorials/multi_crystal_analysis.rst']