<commit_before># Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: process
   :platform: Unix
   :synopsis: Methods for running a chain of plugins

.. moduleauthor:: <NAME> <<EMAIL>>

"""

import os
import logging
import time

from mpi4py import MPI

import savu.plugins.utils as pu


def run_plugin_chain(input_data, plugin_list, processing_dir, mpi=False,
                     processes=1, process=0):
    """Runs a chain of plugins

    :param input_data: The input data to give to the chain
    :type input_data: savu.data.structure.
    :param plugin_list: Names of all the plugins to process in order.
    :type plugin_list: list of str.
    :param processing_dir: Location of the processing directory.
    :type processing_dir: str.
    :param mpi: Whether this is running in mpi, default is false.
    :type mpi: bool.
    """
    in_data = input_data
    output = None
    count = 0
    for plugin_name in plugin_list:
        logging.debug("Loading plugin %s", plugin_name)
        plugin = pu.load_plugin(None, plugin_name)

        # generate somewhere for the data to go
        file_name = os.path.join(processing_dir,
                                 "%02i_%s.h5" % (count, plugin_name))
        output = pu.create_output_data(plugin, in_data, file_name, plugin.name,
                                       mpi)

        plugin.set_parameters(None)

        logging.debug("Starting processing  plugin %s", plugin_name)
        plugin.process(in_data, output, processes, process)
        logging.debug("Completed processing plugin %s", plugin_name)

        in_data.complete()
        in_data = output

        if mpi:
            logging.debug("MPI awaiting barrier")
            MPI.COMM_WORLD.barrier()

        count += 1

    if output is not None:
        output.complete()


def run_process_list(input_data, process_list, processing_dir, mpi=False,
                     processes=1, process=0):
    """Runs a chain of plugins

    :param input_data: The input data to give to the chain
    :type input_data: savu.data.structure.
    :param process_list: Process list.
    :type process_list: savu.data.structure.ProcessList.
    :param processing_dir: Location of the processing directory.
    :type processing_dir: str.
    :param mpi: Whether this is running in mpi, default is false.
    :type mpi: bool.
    """
    filename = os.path.basename(input_data.backing_file.filename)
    filename = os.path.splitext(filename)[0]
    output_filename = \
        os.path.join(processing_dir,
                     "%s_processed_%s.nxs" % (filename,
                                              time.strftime("%Y%m%d%H%M%S")))
    process_list.save_list_to_file(output_filename)
    in_data = input_data
    output = None
    count = 0
    for process_dict in process_list.process_list:
        logging.debug("Loading plugin %s", process_dict['id'])
        plugin = pu.load_plugin(None, process_dict['id'])

        # generate somewhere for the data to go
        file_name = os.path.join(processing_dir,
                                 "%s%02i_%s.h5" % (process_list.name, count,
                                                   process_dict['id']))
        group_name = "%i-%s" % (count, plugin.name)
        output = pu.create_output_data(plugin, in_data, file_name, group_name,
                                       mpi)

        plugin.set_parameters(process_dict['data'])

        logging.debug("Starting processing  plugin %s", process_dict['id'])
        plugin.process(in_data, output, processes, process)
        logging.debug("Completed processing plugin %s", process_dict['id'])

        in_data.complete()
        in_data = output

        if mpi:
            logging.debug("MPI awaiting barrier")
            MPI.COMM_WORLD.barrier()

        if process == 0:
            cite_info = plugin.get_citation_inforamtion()
            if cite_info is not None:
                process_list.add_process_citation(output_filename, count,
                                                  cite_info)
            process_list.add_intermediate_data_link(output_filename,
                                                    output, group_name)

        count += 1

    if output is not None:
        output.complete()<commit_msg>Set up all the files first before running the processing to deal with  mpi requirements<commit_after># Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: process
   :platform: Unix
   :synopsis: Methods for running a chain of plugins

.. moduleauthor:: <NAME> <<EMAIL>>

"""

import os
import logging
import time

from mpi4py import MPI

import savu.plugins.utils as pu


def run_plugin_chain(input_data, plugin_list, processing_dir, mpi=False,
                     processes=1, process=0):
    """Runs a chain of plugins

    :param input_data: The input data to give to the chain
    :type input_data: savu.data.structure.
    :param plugin_list: Names of all the plugins to process in order.
    :type plugin_list: list of str.
    :param processing_dir: Location of the processing directory.
    :type processing_dir: str.
    :param mpi: Whether this is running in mpi, default is false.
    :type mpi: bool.
    """
    in_data = input_data
    output = None
    count = 0
    for plugin_name in plugin_list:
        logging.debug("Loading plugin %s", plugin_name)
        plugin = pu.load_plugin(None, plugin_name)

        # generate somewhere for the data to go
        file_name = os.path.join(processing_dir,
                                 "%02i_%s.h5" % (count, plugin_name))
        output = pu.create_output_data(plugin, in_data, file_name, plugin.name,
                                       mpi)

        plugin.set_parameters(None)

        logging.debug("Starting processing  plugin %s", plugin_name)
        plugin.process(in_data, output, processes, process)
        logging.debug("Completed processing plugin %s", plugin_name)

        in_data.complete()
        in_data = output

        if mpi:
            logging.debug("MPI awaiting barrier")
            MPI.COMM_WORLD.barrier()

        count += 1

    if output is not None:
        output.complete()


def run_process_list(input_data, process_list, processing_dir, mpi=False,
                     processes=1, process=0):
    """Runs a chain of plugins

    :param input_data: The input data to give to the chain
    :type input_data: savu.data.structure.
    :param process_list: Process list.
    :type process_list: savu.data.structure.ProcessList.
    :param processing_dir: Location of the processing directory.
    :type processing_dir: str.
    :param mpi: Whether this is running in mpi, default is false.
    :type mpi: bool.
    """
    filename = os.path.basename(input_data.backing_file.filename)
    filename = os.path.splitext(filename)[0]
    output_filename = \
        os.path.join(processing_dir,
                     "%s_processed_%s.nxs" % (filename,
                                              time.strftime("%Y%m%d%H%M%S")))
    process_list.save_list_to_file(output_filename)
    in_data = input_data
    output = None

    files = []
    count = 0
    for process_dict in process_list.process_list:
        logging.debug("Loading plugin %s", process_dict['id'])
        plugin = pu.load_plugin(None, process_dict['id'])

        # generate somewhere for the data to go
        file_name = os.path.join(processing_dir,
                                 "%s%02i_%s.h5" % (process_list.name, count,
                                                   process_dict['id']))
        group_name = "%i-%s" % (count, plugin.name)
        output = pu.create_output_data(plugin, in_data, file_name, group_name,
                                       mpi)

        files.append(output)

        if process == 0:
            cite_info = plugin.get_citation_inforamtion()
            if cite_info is not None:
                process_list.add_process_citation(output_filename, count,
                                                  cite_info)
            process_list.add_intermediate_data_link(output_filename,
                                                    output, group_name)

        in_data = output
        count += 1

    in_data = input_data
    count = 0
    for process_dict in process_list.process_list:
        logging.debug("Loading plugin %s", process_dict['id'])
        plugin = pu.load_plugin(None, process_dict['id'])

        output = files[count]

        plugin.set_parameters(process_dict['data'])

        logging.debug("Starting processing  plugin %s", process_dict['id'])
        plugin.process(in_data, output, processes, process)
        logging.debug("Completed processing plugin %s", process_dict['id'])

        in_data.complete()
        in_data = output

        count += 1

    if output is not None:
        output.complete()
{'fraction_non_alphanumeric': 0.0881403791851553, 'fraction_numerical': 0.0052440500201694235, 'mean_word_length': 2.296875, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73845', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2702, 'n_words': 830}