<commit_before>"""HDF5 utilities for RAPD"""

"""
This file is part of RAPD

Copyright (C) 2017, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2017-02-08"
_maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Development"

# Standard imports
import argparse
import multiprocessing
import os
import subprocess
import sys
import time

# RAPD imports

VERSIONS = {
    "eiger2cbf": ("160415",)
}

class hdf5_to_cbf_converter(object):

    def __init__(self,
                 master_file,
                 output_dir=False,
                 prefix=False,
                 start_image=False,
                 end_image=False,
                 zfill=5,
                 nproc=False,
                 logger=False):
        """
        Run eiger2cbf on HDF5 dataset. Returns path of new CBF files.
        Not sure I need multiprocessing.Pool, but used as saftety.

        master_file - master file of data to be converted to cbf
        output_dir is output directory
        prefix is new image prefix
        imgn is the image number for the output frame.
        zfill is the number digits for snap image numbers
        returns header
        """

        # from rapd_pilatus import pilatus_read_header as readHeader

        if logger:
            logger.debug("Utilities::convert_hdf5_cbf")

        self.master_file = master_file
        self.output_dir = output_dir
        self.prefix = prefix
        self.start_image = start_image
        self.end_image = end_image
        self.zfill = zfill
        self.nproc = nproc

    def run(self):
        """Coordinates the running of the coonversion process"""

        self.preprocess()
        self.process()

    def preprocess(self):
        """Set up the conversion"""

        # Only master file
        if not "master" in self.master_file:
            raise Exception("Convert needs to be passed a master file")

        # Check directory
        if not self.output_dir:
            self.output_dir = "cbf_files"

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)

        # Check prefix
        if not self.prefix:
            self.prefix = self.master_file.replace("master.h5", "")

        # Check start_image - default to 1
        if not self.start_image:
            self.start_image = 1

        # Check end_image - default to start_image
        # if not self.end_image:
        #     self.end_image = self.start_image

        # Multiprocessing
        if not self.nproc:
            self.nproc = multiprocessing.cpu_count()

    def process(self):
        """Perform the conversion"""

        # The base eiger2cbf command
        command0 = "eiger2cbf %s" % self.master_file

        # Check how many frames are in dataset
        myoutput = subprocess.Popen([command0, self.master_file],
                                    shell=True,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
        stdout, stderr = myoutput.communicate()
        # for i in stdout.split("\n"): print i
        # print stderr
        number_of_images = int(stdout.split("\n")[-2])
        print "Number of images: %d" % number_of_images

        # Work out end image
        if not self.end_image:
            self.end_image = number_of_images + self.start_image - 1
        print "Converting images %d - %d" % (self.start_image, self.end_image)

        # Check that the number of images encompasses the end_image

        # Single image
        if number_of_images == 1:
            img = "%s%s.cbf" % (os.path.join(self.output_dir, self.prefix), str(self.start_image).zfill(self.zfill))
            command = "%s 1 %s" % (command0, img)
            print "Executing command `%s`" % command
            # Now convert
            myoutput = subprocess.Popen(command,
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
            stdout, stderr = myoutput.communicate()

        # Single image from a run of images
        elif self.start_image == self.end_image:
            command = "%s %d:%d %s" % (command0, self.start_image, end, os.path.join(self.output_dir, self.prefix))

            # Now convert
            myoutput = subprocess.Popen(command,
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
            stdout, stderr = myoutput.communicate()

        # Multiple images from a run of images
        else:
            # One processor
            if self.nproc == 1:
                command = "%s %d:%d %s" % (command0, self.start_image, end, os.path.join(self.output_dir, self.prefix))

                # Now convert
                myoutput = subprocess.Popen(command,
                                            shell=True,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.STDOUT)
                stdout, stderr = myoutput.communicate()

            # Multiple processors
            else:

                print "Employing multiple threads"

                # Create function for running
                def run_process(command):
                    """Run the command in a subprocess.Popen call"""
                    print command
                    myoutput = subprocess.Popen(command,
                                                shell=True,
                                                stdout=subprocess.PIPE,
                                                stderr=subprocess.STDOUT)
                    myoutput.wait()
                    stdout, stderr = myoutput.communicate()
                    for i in stdout.split("\n"): print i
                    print stderr
                    return (stdout, stderr)

                # Create a pool to speed things along
                pool = multiprocessing.Pool(processes=self.nproc)

                batch = int(number_of_images / self.nproc)
                iter = 1<commit_msg>Trying to fix<commit_after>"""HDF5 utilities for RAPD"""

"""
This file is part of RAPD

Copyright (C) 2017, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2017-02-08"
_maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Development"

# Standard imports
import argparse
import multiprocessing
import os
import subprocess
import sys
import time

# RAPD imports

VERSIONS = {
    "eiger2cbf": ("160415",)
}

class hdf5_to_cbf_converter(object):

    def __init__(self,
                 master_file,
                 output_dir=False,
                 prefix=False,
                 start_image=False,
                 end_image=False,
                 zfill=5,
                 nproc=False,
                 logger=False):
        """
        Run eiger2cbf on HDF5 dataset. Returns path of new CBF files.
        Not sure I need multiprocessing.Pool, but used as saftety.

        master_file - master file of data to be converted to cbf
        output_dir is output directory
        prefix is new image prefix
        imgn is the image number for the output frame.
        zfill is the number digits for snap image numbers
        returns header
        """

        # from rapd_pilatus import pilatus_read_header as readHeader

        if logger:
            logger.debug("Utilities::convert_hdf5_cbf")

        self.master_file = master_file
        self.output_dir = output_dir
        self.prefix = prefix
        self.start_image = start_image
        self.end_image = end_image
        self.zfill = zfill
        self.nproc = nproc

    def run(self):
        """Coordinates the running of the coonversion process"""

        self.preprocess()
        self.process()

    def preprocess(self):
        """Set up the conversion"""

        # Only master file
        if not "master" in self.master_file:
            raise Exception("Convert needs to be passed a master file")

        # Check directory
        if not self.output_dir:
            self.output_dir = "cbf_files"

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)

        # Check prefix
        if not self.prefix:
            self.prefix = self.master_file.replace("master.h5", "")

        # Check start_image - default to 1
        if not self.start_image:
            self.start_image = 1

        # Check end_image - default to start_image
        # if not self.end_image:
        #     self.end_image = self.start_image

        # Multiprocessing
        if not self.nproc:
            self.nproc = multiprocessing.cpu_count()

    def process(self):
        """Perform the conversion"""

        # The base eiger2cbf command
        command0 = "eiger2cbf %s" % self.master_file

        # Check how many frames are in dataset
        myoutput = subprocess.Popen([command0, self.master_file],
                                    shell=True,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
        stdout, stderr = myoutput.communicate()
        # for i in stdout.split("\n"): print i
        # print stderr
        number_of_images = int(stdout.split("\n")[-2])
        print "Number of images: %d" % number_of_images

        # Work out end image
        if not self.end_image:
            self.end_image = number_of_images + self.start_image - 1
        print "Converting images %d - %d" % (self.start_image, self.end_image)

        # Check that the number of images encompasses the end_image

        # Single image
        if number_of_images == 1:
            img = "%s%s.cbf" % (os.path.join(self.output_dir, self.prefix), str(self.start_image).zfill(self.zfill))
            command = "%s 1 %s" % (command0, img)
            print "Executing command `%s`" % command
            # Now convert
            myoutput = subprocess.Popen(command,
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
            stdout, stderr = myoutput.communicate()

        # Single image from a run of images
        elif self.start_image == self.end_image:
            command = "%s %d:%d %s" % (command0, self.start_image, end, os.path.join(self.output_dir, self.prefix))

            # Now convert
            myoutput = subprocess.Popen(command,
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
            stdout, stderr = myoutput.communicate()

        # Multiple images from a run of images
        else:
            # One processor
            if self.nproc == 1:
                command = "%s %d:%d %s" % (command0, self.start_image, end, os.path.join(self.output_dir, self.prefix))

                # Now convert
                myoutput = subprocess.Popen(command,
                                            shell=True,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.STDOUT)
                stdout, stderr = myoutput.communicate()

            # Multiple processors
            else:

                print "Employing multiple threads"

                # Create function for running
                def run_process(command):
                    """Run the command in a subprocess.Popen call"""
                    print command
                    myoutput = subprocess.Popen(command,
                                                shell=True,
                                                stdout=subprocess.PIPE,
                                                stderr=subprocess.STDOUT)
                    myoutput.wait()
                    stdout, stderr = myoutput.communicate()
                    for i in stdout.split("\n"): print i
                    print stderr
                    return True

                # Create a pool to speed things along
                pool = multiprocessing.Pool(processes=self.nproc)

                batch = int(number_of_images / self.nproc)
                iter = 1
{'fraction_non_alphanumeric': 0.07564959168522643, 'fraction_numerical': 0.0062360801781737195, 'mean_word_length': 1.5767023718439173, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2109538', 'n_tokens_mistral': 3595, 'n_tokens_neox': 3396, 'n_words': 1052}