<commit_before>
#
# Copyright 2014-2015 University of Southern California
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
#

from collections import namedtuple

import numpy as np

ImageMetadata = namedtuple('ImageMetadata', ['x_microns', 'y_microns', 'z_microns', 'axes'])


def plane_distance(p, plane):
    """Return signed distance to plane of point."""
    x, y, z = p
    A, B, C, D = plane
    return A*x + B*y + C*z + D


def bin_reduce(data, axes_s):
    """Reduce ndarray data via bin-averaging for specified per-axis bin sizes.

       For a 3-D input data with shape (D, H, W) and axes_s of [s1, s2,
       s3], the output value result[0, 0, 0] will contain the average
       of all input values in slice data[0:s1, 0:s2, 0:s3] and the result
       will have shape (D/s1, H/s2, W/s3).

       The implementation is optimized for large ndarray data with
       relatively small bin sizes.  It makes a number of Python
       operations proportional to the product of per-axis bin sizes,
       but minimizes the elementwise data access and arithmetic.

       This function converts input data to float32 for averaging and
       it is the caller's responsibility to convert back to a desired
       type.

    """
    d1 = data

    # sort axes by stride distance to optimize for locality
    # doesn't seem to make much difference on modern systems...
    axes = [ (axis, d1.strides[axis]) for axis in range(d1.ndim) ]
    axes.sort(key=lambda p: p[1])

    assert len(axes_s) == data.ndim

    # reduce one axis at a time to shrink work for subsequent axes
    for axis in map(lambda p: p[0], axes):
        s = axes_s[axis]

        if s == 1:
            # skip useless copying for non-reducing axis
            continue

        # accumulate s-strided subsets that belong to each bin
        a = d1[ 
            tuple(
                [ slice(None) for i in range(axis) ]
                + [ slice(0, 1-s, s) ]
                + [ slice(None) for i in range(d1.ndim - axis - 1) ]
            )
        ].astype(np.float32, copy=False)
                
        for step in range(1, s):
            a += d1[ 
                tuple(
                    [ slice(None) for i in range(axis) ]
                    + [ slice(step, step < s and 1-s+step or None, s) ]
                    + [ slice(None) for i in range(d1.ndim - axis - 1) ]
                )
            ]

        # compute single-axis bin averages from accumulation
        d1 = a * (1./s)

    return d1


def load_tiff(fname):
    """Load named file using TIFF reader, returning (data, metadata).

       Data is an ND-array.  Metadata is an ImageMetadata named tuple.

       Raises NotImplementedError if TIFF reader is not available.
       Raises ValueError if file cannot be loaded by TIFF reader.
    """
    try:
        import tifffile
        from xml.dom import minidom
    except Exception, te:
        raise NotImplementedError(str(te))

    tf = tifffile.TiffFile(fname, multifile=False)
    data = None
    md = None

    def canonicalize(data, axes, flipY=True):
        """Restructure to our preferred CZYX format."""
        print data.shape, axes, data.dtype, data.strides, data.nbytes

        pos = axes.find('T')
        if pos >= 0:
            # remove useless time axis
            if data.shape[pos] == 1:
                data = data[
                    tuple(
                        [ slice(None) for i in range(pos) ]
                        + [ 0 ]
                        + [ slice(None) for i in range(pos+1, data.ndim) ]
                    )
                ]
                axes = axes[0:pos] + axes[pos+1:]
            else:
                raise NotImplementedError('Unexpected TIFF with T axis length %d' % data.shape[pos])

        pos = axes.find('C')
        if pos >= 1:
            # move color axis to first position
            data = data.transpose(
                *tuple(
                    [ pos ]
                    + [ i for i in range(data.ndim) if i != pos ]
                )
            )
            axes = axes[pos] + ''.join([
                axes[i] for i in range(data.ndim) if i != pos
            ])
        elif pos == 0:
            pass
        elif data.ndim == 3:
            pass
        else:
            raise NotImplementedError('Unexpected %d-dimension TIFF without C axis.' % data.ndim)

        # flip Y axis to match Fiji user expectation
        pos = axes.find('Y')
        if pos >= 0:
            if flipY:
                data = data[
                    tuple(
                        [ slice(None) for i in range(pos) ]
                        + [ slice(None, None, -1) ]
                        + [ slice(None) for i in range(pos+1, data.ndim) ]
                    )
                ]
        else:
            raise NotImplementedError('Unexpected TIFF without Y axis.')

        print data.shape, axes, data.dtype, data.strides, data.nbytes
        return data, axes<commit_msg>add trivial 'C' axis for single-channel images<commit_after>
#
# Copyright 2014-2015 University of Southern California
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
#

from collections import namedtuple

import numpy as np

ImageMetadata = namedtuple('ImageMetadata', ['x_microns', 'y_microns', 'z_microns', 'axes'])


def plane_distance(p, plane):
    """Return signed distance to plane of point."""
    x, y, z = p
    A, B, C, D = plane
    return A*x + B*y + C*z + D


def bin_reduce(data, axes_s):
    """Reduce ndarray data via bin-averaging for specified per-axis bin sizes.

       For a 3-D input data with shape (D, H, W) and axes_s of [s1, s2,
       s3], the output value result[0, 0, 0] will contain the average
       of all input values in slice data[0:s1, 0:s2, 0:s3] and the result
       will have shape (D/s1, H/s2, W/s3).

       The implementation is optimized for large ndarray data with
       relatively small bin sizes.  It makes a number of Python
       operations proportional to the product of per-axis bin sizes,
       but minimizes the elementwise data access and arithmetic.

       This function converts input data to float32 for averaging and
       it is the caller's responsibility to convert back to a desired
       type.

    """
    d1 = data

    # sort axes by stride distance to optimize for locality
    # doesn't seem to make much difference on modern systems...
    axes = [ (axis, d1.strides[axis]) for axis in range(d1.ndim) ]
    axes.sort(key=lambda p: p[1])

    assert len(axes_s) == data.ndim

    # reduce one axis at a time to shrink work for subsequent axes
    for axis in map(lambda p: p[0], axes):
        s = axes_s[axis]

        if s == 1:
            # skip useless copying for non-reducing axis
            continue

        # accumulate s-strided subsets that belong to each bin
        a = d1[ 
            tuple(
                [ slice(None) for i in range(axis) ]
                + [ slice(0, 1-s, s) ]
                + [ slice(None) for i in range(d1.ndim - axis - 1) ]
            )
        ].astype(np.float32, copy=False)
                
        for step in range(1, s):
            a += d1[ 
                tuple(
                    [ slice(None) for i in range(axis) ]
                    + [ slice(step, step < s and 1-s+step or None, s) ]
                    + [ slice(None) for i in range(d1.ndim - axis - 1) ]
                )
            ]

        # compute single-axis bin averages from accumulation
        d1 = a * (1./s)

    return d1


def load_tiff(fname):
    """Load named file using TIFF reader, returning (data, metadata).

       Data is an ND-array.  Metadata is an ImageMetadata named tuple.

       Raises NotImplementedError if TIFF reader is not available.
       Raises ValueError if file cannot be loaded by TIFF reader.
    """
    try:
        import tifffile
        from xml.dom import minidom
    except Exception, te:
        raise NotImplementedError(str(te))

    tf = tifffile.TiffFile(fname, multifile=False)
    data = None
    md = None

    def canonicalize(data, axes, flipY=True):
        """Restructure to our preferred CZYX format."""
        print data.shape, axes, data.dtype, data.strides, data.nbytes

        pos = axes.find('T')
        if pos >= 0:
            # remove useless time axis
            if data.shape[pos] == 1:
                data = data[
                    tuple(
                        [ slice(None) for i in range(pos) ]
                        + [ 0 ]
                        + [ slice(None) for i in range(pos+1, data.ndim) ]
                    )
                ]
                axes = axes[0:pos] + axes[pos+1:]
            else:
                raise NotImplementedError('Unexpected TIFF with T axis length %d' % data.shape[pos])

        pos = axes.find('C')
        if pos >= 1:
            # move color axis to first position
            data = data.transpose(
                *tuple(
                    [ pos ]
                    + [ i for i in range(data.ndim) if i != pos ]
                )
            )
            axes = axes[pos] + ''.join([
                axes[i] for i in range(data.ndim) if i != pos
            ])
        elif pos == 0:
            pass
        elif data.ndim == 3:
            data = data[None,...]
            axes = 'C' + axes
        else:
            raise NotImplementedError('Unexpected %d-dimension TIFF without C axis.' % data.ndim)

        # flip Y axis to match Fiji user expectation
        pos = axes.find('Y')
        if pos >= 0:
            if flipY:
                data = data[
                    tuple(
                        [ slice(None) for i in range(pos) ]
                        + [ slice(None, None, -1) ]
                        + [ slice(None) for i in range(pos+1, data.ndim) ]
                    )
                ]
        else:
            raise NotImplementedError('Unexpected TIFF without Y axis.')

        print data.shape, axes, data.dtype, data.strides, data.nbytes
        return data, axes
{'fraction_non_alphanumeric': 0.09391824526420738, 'fraction_numerical': 0.011764705882352941, 'mean_word_length': 1.5960144927536233, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2524495', 'n_tokens_mistral': 3065, 'n_tokens_neox': 2805, 'n_words': 1041}