<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2014, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import fiona
from fiona.transform import transform_geom
import numpy
import pandas
import rasterio
from rasterio.crs import is_same_crs
from rasterio.features import rasterize
import rtree
from shapely.geometry import shape as shp
from idl_functions import histogram
from idl_functions import array_indices


class Segments(object):

    """
    Given a segmented array, the Segments class will find the segments and
    optionally calculate basic statistics. A value of zero is considered
    to be the background and ignored.

    Designed as an easier interface for analysing segmented regions.

    Example:

        >>> seg_array = numpy.zeros((10,10), dtype='uint8')
        >>> seg_array[0:3,0:3] = 1
        >>> seg_array[0:3,7:10] = 2
        >>> seg_array[7:10,0:3] = 3
        >>> seg_array[7:10,7:10] = 4
        >>> seg_ds = Segments(seg_array)
        >>> vals = numpy.arange(100).reshape((10,10))
        >>> seg_ds.mean(vals)
        >>> seg_ds.max(vals)
        >>> seg_ds.min(vals)
        >>> seg_ds.data(vals, segment_id=2)
        >>> seg_ds.locations(segment_id=3)
    """

    def __init__(self, array):
        """
        Initialises the Segments class.

        :param array:
            A 2D NumPy array containing the segmented array.
        """
        if array.ndim != 2:
            msg = "Dimensions of array must be 2D! Supplied array is {dims}"
            msg = msg.format(dims=array.ndim)
            raise Exception(msg)

        self.array = array
        self.array_1D = array.ravel()

        self.dims = array.shape

        self.histogram = None
        self.ri = None

        self.min_id = None
        self.max_id = None
        self.n_segments = None
        self.ids = None

        self._find_segments()


    def _find_segments(self):
        """
        Determines the pixel locations for every segment/region contained
        within a 2D array. The minimum and maximum segemnt ID's/labels are
        also determined.
        """
        h = histogram(self.array_1D, minv=0, reverse_indices='ri')

        self.histogram = h['histogram']
        self.ri = h['ri']

        # Determine the min and max segment ID's
        mx = numpy.max(self.array)
        if mx > 0:
            mn = numpy.min(self.array[self.array > 0])
        else:
            mn = mx

        self.min_id = mn
        self.max_id = mx

        # Determine the segment ids and the number of segments
        self.ids = numpy.where(self.histogram > 0)[0][1:]
        self.n_segments = self.ids.shape[0]


    def data(self, array, segment_id=1):
        """
        Retrieve the data from an array corresponding to a segment_id.

        :param array:
            A 2D NumPy array containing the data to be extracted given
            a segment_id.

        :param segment_id:
            An integer corresponding to the segment_id of interest.
            Default is the first segment.

        :return:
            A 1D NumPy array containing the data from array corresponding
            to the segment_id. If no segment exists, then an empty array
            is returned.
        """
        ri = self.ri
        i = segment_id
        arr_flat = array.ravel()

        # Check for bounds
        if (i > self.max_id) or (i < self.min_id):
            data = numpy.array([])
            return data
<commit_msg>Re-worked the min/max and seg id's list. Also included the include_zero paramater.<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2014, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import fiona
from fiona.transform import transform_geom
import numpy
import pandas
import rasterio
from rasterio.crs import is_same_crs
from rasterio.features import rasterize
import rtree
from shapely.geometry import shape as shp
from idl_functions import histogram
from idl_functions import array_indices

# TODO: calculate the seg ids something like this
"""
x = numpy.arange(seg.histogram.shape[0])

x
array([    0,     1,     2, ...,  9998,  9999, 10000])

x[wh]
array([    1,     2,     3, ...,  9998,  9999, 10000])
"""



class Segments(object):

    """
    Given a segmented array, the Segments class will find the segments and
    optionally calculate basic statistics. A value of zero is considered
    to be the background and ignored, unless `include_zero` is set to
    True.

    Designed as an easier interface for analysing segmented regions.

    Example:

        >>> seg_array = numpy.zeros((10,10), dtype='uint8')
        >>> seg_array[0:3,0:3] = 1
        >>> seg_array[0:3,7:10] = 2
        >>> seg_array[7:10,0:3] = 3
        >>> seg_array[7:10,7:10] = 4
        >>> seg_ds = Segments(seg_array)
        >>> vals = numpy.arange(100).reshape((10,10))
        >>> seg_ds.mean(vals)
        >>> seg_ds.max(vals)
        >>> seg_ds.min(vals)
        >>> seg_ds.data(vals, segment_id=2)
        >>> seg_ds.locations(segment_id=3)
    """

    def __init__(self, array, include_zero=False):
        """
        Initialises the Segments class.

        :param array:
            A 2D NumPy array containing the segmented array.

        :param include_zero:
            A `bool` indicating whether or not to include the
            value of zero as a valid segment id, rather than
            ignore it.
            Default is False, ignore the value of zero.
        """
        if array.ndim != 2:
            msg = "Dimensions of array must be 2D! Supplied array is {dims}"
            msg = msg.format(dims=array.ndim)
            raise Exception(msg)

        self.array = array
        self.array_1D = array.ravel()

        self.dims = array.shape

        self.histogram = None
        self.ri = None

        self.min_id = None
        self.max_id = None
        self.n_segments = None
        self.ids = None

        self._find_segments(include_zero=include_zero)


    def _find_segments(self, include_zero=False):
        """
        Determines the pixel locations for every segment/region contained
        within a 2D array. The minimum and maximum segemnt ID's/labels are
        also determined.
        """
        h = histogram(self.array_1D, minv=0, reverse_indices='ri')

        self.histogram = h['histogram']
        self.ri = h['ri']

        # Determine the segment ids
        idx = self.histogram != 0
        ids = numpy.arange(self.histogram.shape[0])[idx]

        # include or exclude the segment id of 0
        if include_zero:
            self.ids = ids
        else:
            if ids[0] == 0:
                self.ids = ids[1:]
            else:
                self.ids = ids

        # min & max segment id
        self.min_id = self.ids[0]
        self.max_id = self.ids[-1]

        # number of segments
        self.n_segments = self.ids.shape[0]


    def data(self, array, segment_id=1):
        """
        Retrieve the data from an array corresponding to a segment_id.

        :param array:
            A 2D NumPy array containing the data to be extracted given
            a segment_id.

        :param segment_id:
            An integer corresponding to the segment_id of interest.
            Default is the first segment.

        :return:
            A 1D NumPy array containing the data from array corresponding
            to the segment_id. If no segment exists, then an empty array
            is returned.
        """
        ri = self.ri
        i = segment_id
        arr_flat = array.ravel()

        # Check for bounds
        if (i > self.max_id) or (i < self.min_id):
            data = numpy.array([])
            return data

{'fraction_non_alphanumeric': 0.07869728766580875, 'fraction_numerical': 0.01504652544050683, 'mean_word_length': 2.4551983584131327, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5858908', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2752, 'n_words': 1077}