<commit_before># Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: tomo_recon_mpi
   :platform: Unix
   :synopsis: runner for full reconstruction pilpeline using the MPI framework

.. moduleauthor:: <NAME> <<EMAIL>>

"""
import logging
import optparse
import socket
import sys
import os

from itertools import chain
from mpi4py import MPI

from savu.core import process
from savu.data.process_data import ProcessList

import savu.plugins.utils as pu

if __name__ == '__main__':

    usage = "%prog [options] nxs_file_to_reconstruct process_file output_dir"
    version = "%prog 0.1"
    parser = optparse.OptionParser(usage=usage, version=version)
    parser.add_option("-n", "--names", dest="names", help="Process names",
                      default="CPU1,CPU2,CPU3,CPU4,CPU5,CPU6,CPU7,CPU8",
                      type='string')
    (options, args) = parser.parse_args()

    # Check basic items for completeness
    if len(args) is not 3:
        print "filename, process file and output path needs to be specified"
        print "Exiting with error code 1 - incorrect number of inputs"
        sys.exit(1)

    if not os.path.exists(args[0]):
        print("Input file '%s' does not exist" % args[0])
        print("Exiting with error code 2 - Input file missing")
        sys.exit(2)

    if not os.path.exists(args[1]):
        print("Processing file '%s' does not exist" % args[1])
        print("Exiting with error code 3 - Processing file missing")
        sys.exit(3)

    if not os.path.exists(args[2]):
        print("Output Directory '%s' does not exist" % args[2])
        print("Exiting with error code 4 - Output Directory missing")
        sys.exit(4)

    RANK_NAMES = options.names.split(',')

    RANK = MPI.COMM_WORLD.rank
    SIZE = MPI.COMM_WORLD.size
    RANK_NAMES_SIZE = len(RANK_NAMES)
    if RANK_NAMES_SIZE > SIZE:
        RANK_NAMES_SIZE = SIZE
    MACHINES = SIZE/RANK_NAMES_SIZE
    MACHINE_RANK = RANK/MACHINES
    MACHINE_RANK_NAME = RANK_NAMES[MACHINE_RANK]
    MACHINE_NUMBER = RANK % MACHINES
    MACHINE_NUMBER_STRING = "%03i" % (MACHINE_NUMBER)
    ALL_PROCESSES = [[i]*MACHINES for i in RANK_NAMES]
    ALL_PROCESSES = list(chain.from_iterable(ALL_PROCESSES))

    logging.basicConfig(level=0, format='L %(relativeCreated)12d M' +
                        MACHINE_NUMBER_STRING + ' ' + MACHINE_RANK_NAME +
                        ' %(levelname)-6s %(message)s', datefmt='%H:%M:%S')

    MPI.COMM_WORLD.barrier()

    logging.info("Starting the reconstruction pipeline process")

    logging.debug("Rank : %i - Size : %i", RANK, SIZE)

    IP = socket.gethostbyname(socket.gethostname())

    logging.debug("ip address is : %s", IP)

    MPI.COMM_WORLD.barrier()

    logging.debug("LD_LIBRARY_PATH is %s",  os.getenv('LD_LIBRARY_PATH'))

    MPI.COMM_WORLD.barrier()

    process_filename = options.process_filename

    process_list = ProcessList()
    process_list.populate_process_list(process_filename)

    first_plugin = pu.load_plugin(process_list.process_list[0]['id'])
    input_data = pu.load_raw_data(args[0])

    process.run_process_list(input_data, process_list, args[1],
                             mpi=True, processes=ALL_PROCESSES,
                             process=RANK)

    MPI.COMM_WORLD.barrier()

    logging.info("Python MPI script complete")<commit_msg>Fixed up bug with changing from options to args in main runner<commit_after># Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: tomo_recon_mpi
   :platform: Unix
   :synopsis: runner for full reconstruction pilpeline using the MPI framework

.. moduleauthor:: <NAME> <<EMAIL>>

"""
import logging
import optparse
import socket
import sys
import os

from itertools import chain
from mpi4py import MPI

from savu.core import process
from savu.data.process_data import ProcessList

import savu.plugins.utils as pu

if __name__ == '__main__':

    usage = "%prog [options] nxs_file_to_reconstruct process_file output_dir"
    version = "%prog 0.1"
    parser = optparse.OptionParser(usage=usage, version=version)
    parser.add_option("-n", "--names", dest="names", help="Process names",
                      default="CPU1,CPU2,CPU3,CPU4,CPU5,CPU6,CPU7,CPU8",
                      type='string')
    (options, args) = parser.parse_args()

    # Check basic items for completeness
    if len(args) is not 3:
        print "filename, process file and output path needs to be specified"
        print "Exiting with error code 1 - incorrect number of inputs"
        sys.exit(1)

    if not os.path.exists(args[0]):
        print("Input file '%s' does not exist" % args[0])
        print("Exiting with error code 2 - Input file missing")
        sys.exit(2)

    if not os.path.exists(args[1]):
        print("Processing file '%s' does not exist" % args[1])
        print("Exiting with error code 3 - Processing file missing")
        sys.exit(3)

    if not os.path.exists(args[2]):
        print("Output Directory '%s' does not exist" % args[2])
        print("Exiting with error code 4 - Output Directory missing")
        sys.exit(4)

    RANK_NAMES = options.names.split(',')

    RANK = MPI.COMM_WORLD.rank
    SIZE = MPI.COMM_WORLD.size
    RANK_NAMES_SIZE = len(RANK_NAMES)
    if RANK_NAMES_SIZE > SIZE:
        RANK_NAMES_SIZE = SIZE
    MACHINES = SIZE/RANK_NAMES_SIZE
    MACHINE_RANK = RANK/MACHINES
    MACHINE_RANK_NAME = RANK_NAMES[MACHINE_RANK]
    MACHINE_NUMBER = RANK % MACHINES
    MACHINE_NUMBER_STRING = "%03i" % (MACHINE_NUMBER)
    ALL_PROCESSES = [[i]*MACHINES for i in RANK_NAMES]
    ALL_PROCESSES = list(chain.from_iterable(ALL_PROCESSES))

    logging.basicConfig(level=0, format='L %(relativeCreated)12d M' +
                        MACHINE_NUMBER_STRING + ' ' + MACHINE_RANK_NAME +
                        ' %(levelname)-6s %(message)s', datefmt='%H:%M:%S')

    MPI.COMM_WORLD.barrier()

    logging.info("Starting the reconstruction pipeline process")

    logging.debug("Rank : %i - Size : %i", RANK, SIZE)

    IP = socket.gethostbyname(socket.gethostname())

    logging.debug("ip address is : %s", IP)

    MPI.COMM_WORLD.barrier()

    logging.debug("LD_LIBRARY_PATH is %s",  os.getenv('LD_LIBRARY_PATH'))

    MPI.COMM_WORLD.barrier()

    process_filename = args[1]

    process_list = ProcessList()
    process_list.populate_process_list(process_filename)

    first_plugin = pu.load_plugin(process_list.process_list[0]['id'])
    input_data = pu.load_raw_data(args[0])

    process.run_process_list(input_data, process_list, args[2],
                             mpi=True, processes=ALL_PROCESSES,
                             process=RANK)

    MPI.COMM_WORLD.barrier()

    logging.info("Python MPI script complete")
{'fraction_non_alphanumeric': 0.10493192910351913, 'fraction_numerical': 0.011173901875160545, 'mean_word_length': 3.2114656571119524, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3588464', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2418, 'n_words': 676}