<commit_before>#!/usr/bin/env python

__author__ = "<NAME>"
__date__ = "$Apr 9, 2014 4:00:40PM$"


import os
import sys
import json
import itertools
import multiprocessing
import subprocess
import time

# Generally useful and fast to import so done immediately.
import numpy

import h5py

import lazyflow.utility.pathHelpers

# Need in order to have logging information no matter what.
import debugging_tools

import generic_decorators

import expanded_numpy

import additional_generators

import HDF5_recorder

# Short function to process image data.
import advanced_image_processing

# For IO. Right now, just includes read_parameters for reading a config.json file.
import read_config

import HDF5_serializers


# Get the logger
logger = debugging_tools.logging.getLogger(__name__)



@debugging_tools.log_call(logger)
def generate_neurons_io_handler(input_filename, output_filename, parameters_filename):
    """
        Uses generate_neurons to process a input_filename (HDF5 dataset) and outputs results to an output_filename (HDF5
        dataset). Also,

        Args:
            input_filename          HDF5 filename to read from (should be a path to a h5py.Dataset)
            output_filename         HDF5 filename to write to (should be a path to a h5py.Group)
            parameters_filename     JSON filename with parameters.
    """


    # Extract and validate file extensions.

    # Parse parameter filename and validate that the name is acceptable
    parameters_filename_details = lazyflow.utility.pathHelpers.PathComponents(parameters_filename)
    # Clean up the extension so it fits the standard.
    if ( parameters_filename_details.extension.lower().lstrip(os.extsep) not in ["json"] ):
        raise Exception("Parameter file with filename: \"" + parameters_filename + "\"" + " provided with an unknown file extension: \"" + parameters_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse the parameters from the json file.
    parameters = read_config.read_parameters(parameters_filename)

    if (len(parameters) == 1) and ("generate_neurons_blocks" in parameters):
        generate_neurons_blocks(input_filename, output_filename, **parameters["generate_neurons_blocks"])
    else:
        generate_neurons_a_block(input_filename, output_filename, **parameters)


@debugging_tools.log_call(logger)
def generate_neurons_a_block(input_filename, output_filename, debug = False, **parameters):
    """
        Uses generate_neurons to process a input_filename (HDF5 dataset) and outputs results to an output_filename (HDF5
        dataset). Also,

        Args:
            input_filename          HDF5 filename to read from (should be a path to a h5py.Dataset)
            output_filename         HDF5 filename to write to (should be a path to a h5py.Group)
            parameters              how the run should be configured.
    """


    # Extract and validate file extensions.

    # Parse input filename and validate that the name is acceptable
    input_filename_details = lazyflow.utility.pathHelpers.PathComponents(input_filename)
    # Clean up the extension so it fits the standard.
    if ( input_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Input file with filename: \"" + input_filename + "\"" + " provided with an unknown file extension: \"" + input_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse output filename and validate that the name is acceptable
    output_filename_details = lazyflow.utility.pathHelpers.PathComponents(output_filename)
    # Clean up the extension so it fits the standard.
    if ( output_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Output file with filename: \"" + output_filename + "\"" + " provided with an unknown file extension: \"" + output_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Where the original images are.
    input_dataset_name = input_filename_details.internalPath

    # Name of the group where all data will be stored.
    output_group_name = output_filename_details.internalPath


    # Read the input data.
    original_images = None
    with h5py.File(input_filename_details.externalPath, "r") as input_file_handle:
        original_images = HDF5_serializers.read_numpy_structured_array_from_HDF5(input_file_handle, input_dataset_name)
        original_images = original_images.astype(numpy.float32)

    # Write out the output.
    with h5py.File(output_filename_details.externalPath, "a") as output_file_handle:
        # Create a new output directory if doesn't exists.
        if output_group_name not in output_file_handle:
            output_file_handle.create_group(output_group_name)

        # Group where all data will be stored.
        output_group = output_file_handle[output_group_name]

        # Create a soft link to the original images. But use the appropriate type of soft link depending on whether
        # the input and output file are the same.
        if "original_images" not in output_group:
            if input_filename_details.externalPath == output_filename_details.externalPath:
                output_group["original_images"] = h5py.SoftLink(input_dataset_name)
            else:
                output_group["original_images"] = h5py.ExternalLink(input_filename_details.externalPath, input_dataset_name)

        # Get a debug logger for the HDF5 file (if needed)
        array_debug_recorder = HDF5_recorder.generate_HDF5_array_recorder(output_group,
            group_name = "debug",
            enable = debug,
            overwrite_group = False,
            recorder_constructor = HDF5_recorder.HDF5EnumeratedArrayRecorder
        )

        # Saves intermediate result to make resuming easier
        resume_logger = HDF5_recorder.generate_HDF5_array_recorder(output_group,
            recorder_constructor = HDF5_recorder.HDF5ArrayRecorder,
            overwrite = True
        )

        # Generate the neurons and attempt to resume if possible
        generate_neurons.resume_logger = resume_logger
        generate_neurons.recorders.array_debug_recorder = array_debug_recorder
        generate_neurons(original_images = original_images, **parameters["generate_neurons"])

        # Save the configuration parameters in the attributes as a string.
        if "parameters" not in output_group.attrs:
            # Write the configuration parameters in the attributes as a string.
            output_group.attrs["parameters"] = repr(dict(list(parameters.items()) + \
                                                         [("debug", debug)]
                                                        )
            )


@debugging_tools.log_call(logger)
def generate_neurons_blocks(input_filename, output_filename, num_processes = multiprocessing.cpu_count(), block_shape = None, num_blocks = None, half_window_shape = None, half_border_shape = None, use_drmaa = False, num_drmaa_cores = 16, debug = False, **parameters):
    #TODO: Move this function into a new module with its own command line interface.
    #TODO: Heavy refactoring required on this function.

    # Extract and validate file extensions.

    # Parse input filename and validate that the name is acceptable
    input_filename_details = lazyflow.utility.pathHelpers.PathComponents(input_filename)
    # Clean up the extension so it fits the standard.
    if ( input_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Input file with filename: \"" + input_filename + "\"" + " provided with an unknown file extension: \"" + input_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse output filename and validate that the name is acceptable
    output_filename_details = lazyflow.utility.pathHelpers.PathComponents(output_filename)
    # Clean up the extension so it fits the standard.
    if ( output_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Output file with filename: \"" + output_filename + "\"" + " provided with an unknown file extension: \"" + output_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Where the original images are.
    input_dataset_name = input_filename_details.internalPath

    # Name of the group where all data will be stored.
    output_group_name = output_filename_details.internalPath
<commit_msg>nanshe/nanshe/nanshe_learner.py: Switch path to use the local copy of `pathHelpers`.<commit_after>#!/usr/bin/env python

__author__ = "<NAME>"
__date__ = "$Apr 9, 2014 4:00:40PM$"


import os
import sys
import json
import itertools
import multiprocessing
import subprocess
import time

# Generally useful and fast to import so done immediately.
import numpy

import h5py

import pathHelpers

# Need in order to have logging information no matter what.
import debugging_tools

import generic_decorators

import expanded_numpy

import additional_generators

import HDF5_recorder

# Short function to process image data.
import advanced_image_processing

# For IO. Right now, just includes read_parameters for reading a config.json file.
import read_config

import HDF5_serializers


# Get the logger
logger = debugging_tools.logging.getLogger(__name__)



@debugging_tools.log_call(logger)
def generate_neurons_io_handler(input_filename, output_filename, parameters_filename):
    """
        Uses generate_neurons to process a input_filename (HDF5 dataset) and outputs results to an output_filename (HDF5
        dataset). Also,

        Args:
            input_filename          HDF5 filename to read from (should be a path to a h5py.Dataset)
            output_filename         HDF5 filename to write to (should be a path to a h5py.Group)
            parameters_filename     JSON filename with parameters.
    """


    # Extract and validate file extensions.

    # Parse parameter filename and validate that the name is acceptable
    parameters_filename_details = pathHelpers.PathComponents(parameters_filename)
    # Clean up the extension so it fits the standard.
    if ( parameters_filename_details.extension.lower().lstrip(os.extsep) not in ["json"] ):
        raise Exception("Parameter file with filename: \"" + parameters_filename + "\"" + " provided with an unknown file extension: \"" + parameters_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse the parameters from the json file.
    parameters = read_config.read_parameters(parameters_filename)

    if (len(parameters) == 1) and ("generate_neurons_blocks" in parameters):
        generate_neurons_blocks(input_filename, output_filename, **parameters["generate_neurons_blocks"])
    else:
        generate_neurons_a_block(input_filename, output_filename, **parameters)


@debugging_tools.log_call(logger)
def generate_neurons_a_block(input_filename, output_filename, debug = False, **parameters):
    """
        Uses generate_neurons to process a input_filename (HDF5 dataset) and outputs results to an output_filename (HDF5
        dataset). Also,

        Args:
            input_filename          HDF5 filename to read from (should be a path to a h5py.Dataset)
            output_filename         HDF5 filename to write to (should be a path to a h5py.Group)
            parameters              how the run should be configured.
    """


    # Extract and validate file extensions.

    # Parse input filename and validate that the name is acceptable
    input_filename_details = pathHelpers.PathComponents(input_filename)
    # Clean up the extension so it fits the standard.
    if ( input_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Input file with filename: \"" + input_filename + "\"" + " provided with an unknown file extension: \"" + input_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse output filename and validate that the name is acceptable
    output_filename_details = pathHelpers.PathComponents(output_filename)
    # Clean up the extension so it fits the standard.
    if ( output_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Output file with filename: \"" + output_filename + "\"" + " provided with an unknown file extension: \"" + output_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Where the original images are.
    input_dataset_name = input_filename_details.internalPath

    # Name of the group where all data will be stored.
    output_group_name = output_filename_details.internalPath


    # Read the input data.
    original_images = None
    with h5py.File(input_filename_details.externalPath, "r") as input_file_handle:
        original_images = HDF5_serializers.read_numpy_structured_array_from_HDF5(input_file_handle, input_dataset_name)
        original_images = original_images.astype(numpy.float32)

    # Write out the output.
    with h5py.File(output_filename_details.externalPath, "a") as output_file_handle:
        # Create a new output directory if doesn't exists.
        if output_group_name not in output_file_handle:
            output_file_handle.create_group(output_group_name)

        # Group where all data will be stored.
        output_group = output_file_handle[output_group_name]

        # Create a soft link to the original images. But use the appropriate type of soft link depending on whether
        # the input and output file are the same.
        if "original_images" not in output_group:
            if input_filename_details.externalPath == output_filename_details.externalPath:
                output_group["original_images"] = h5py.SoftLink(input_dataset_name)
            else:
                output_group["original_images"] = h5py.ExternalLink(input_filename_details.externalPath, input_dataset_name)

        # Get a debug logger for the HDF5 file (if needed)
        array_debug_recorder = HDF5_recorder.generate_HDF5_array_recorder(output_group,
            group_name = "debug",
            enable = debug,
            overwrite_group = False,
            recorder_constructor = HDF5_recorder.HDF5EnumeratedArrayRecorder
        )

        # Saves intermediate result to make resuming easier
        resume_logger = HDF5_recorder.generate_HDF5_array_recorder(output_group,
            recorder_constructor = HDF5_recorder.HDF5ArrayRecorder,
            overwrite = True
        )

        # Generate the neurons and attempt to resume if possible
        generate_neurons.resume_logger = resume_logger
        generate_neurons.recorders.array_debug_recorder = array_debug_recorder
        generate_neurons(original_images = original_images, **parameters["generate_neurons"])

        # Save the configuration parameters in the attributes as a string.
        if "parameters" not in output_group.attrs:
            # Write the configuration parameters in the attributes as a string.
            output_group.attrs["parameters"] = repr(dict(list(parameters.items()) + \
                                                         [("debug", debug)]
                                                        )
            )


@debugging_tools.log_call(logger)
def generate_neurons_blocks(input_filename, output_filename, num_processes = multiprocessing.cpu_count(), block_shape = None, num_blocks = None, half_window_shape = None, half_border_shape = None, use_drmaa = False, num_drmaa_cores = 16, debug = False, **parameters):
    #TODO: Move this function into a new module with its own command line interface.
    #TODO: Heavy refactoring required on this function.

    # Extract and validate file extensions.

    # Parse input filename and validate that the name is acceptable
    input_filename_details = pathHelpers.PathComponents(input_filename)
    # Clean up the extension so it fits the standard.
    if ( input_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Input file with filename: \"" + input_filename + "\"" + " provided with an unknown file extension: \"" + input_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Parse output filename and validate that the name is acceptable
    output_filename_details = pathHelpers.PathComponents(output_filename)
    # Clean up the extension so it fits the standard.
    if ( output_filename_details.extension.lower().lstrip(os.extsep) not in ["h5", "hdf5", "he5"] ):
        raise Exception("Output file with filename: \"" + output_filename + "\"" + " provided with an unknown file extension: \"" + output_filename_details.extension + "\". If it is a supported format, please run the given file through nanshe_converter first before proceeding.")

    # Where the original images are.
    input_dataset_name = input_filename_details.internalPath

    # Name of the group where all data will be stored.
    output_group_name = output_filename_details.internalPath

{'fraction_non_alphanumeric': 0.07103825136612021, 'fraction_numerical': 0.0064221733986817645, 'mean_word_length': 3.4850934815563415, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7225524', 'n_tokens_mistral': 4739, 'n_tokens_neox': 4635, 'n_words': 1543}