<commit_before>import numpy as np
import xarray as xr
from .clean_mask import create_circular_mask
from skimage.filters.rank import modal
from skimage.morphology import remove_small_objects
import scipy

## Selective Filters (do not necessarily apply to all pixels) ##

def lone_object_filter(image, min_size=2, connectivity=1, kernel_size=3):
    """
    Replaces isolated, contiguous regions of values in a raster with values
    representing the surrounding pixels.

    More specifically, this reduces noise in a raster by setting
    contiguous regions of values greater than a specified minimum size to
    the modal value in a specified neighborhood.

    The default argument values filter out lone, singular pixels.
    This filter is not idempotent, so it may need to be applied repeatedly
    until the output stops changing or the results are acceptable.

    Args:
        image (numpy.ndarray):
            The image to filter. Must not contain NaNs.
        min_size (int):
            Defines the minimum number of contiguous pixels that will not
            be set to the modal value of their neighborhood. Must be greater than 2.
            Setting this to 1 is pointless, since this function will then do nothing to the raster.
        connectivity (int):
            The maximum distance between any two pixels such that they are
            considered one group. For example, a connectivity of 1 considers
            only adjacent values to be within one group, but a connectivity of 2
            also considers diagonally connected values to be within one group. 
            Must be greater than 0.
        kernel_size (int or float):
            The diameter of the circular kernel to use for the modal filter.
            If there are still contiguous regions of pixels that should be set to the
            modal value of their neighborhood, increase this value to remove them.
            This parameter generally should scale roughly with the square root of the 
            `min_size` parameter. Note that the larger this value is, the more detail 
            may be lost in the image and the slower this function will run.

    Returns:
        The filtered image.

    Authors:
        <NAME> (<EMAIL>)\n
        <NAME>    (<EMAIL>)
    """
    assert kernel_size % 2 == 1, "The parameter `kernel_size` must be an odd number."
    image_min, image_max = image.min(), image.max()
    image_dtype = image.dtype
    image = np.interp(image, [image_min, image_max], [0,255]).astype(np.uint8)
    modal_filtered = modal(image, create_circular_mask(kernel_size, kernel_size))
    
    da = xr.DataArray(image)
    for i, val in enumerate(np.unique(image)):
        # Determine the pixels with this value that will be not be filtered (True to keep).
        layer = remove_small_objects(image == val, min_size=min_size, connectivity=connectivity)
        # Select the values from the image that will remain (filter it).
        filtered = da.where(layer) if i == 0 else filtered.combine_first(da.where(layer))
    # Fill in the removed values with their local modes.<commit_msg>raster_filter.py: lone_object_filter(): Removed incorrect statement regarding kernel_size parameter.<commit_after>import numpy as np
import xarray as xr
from .clean_mask import create_circular_mask
from skimage.filters.rank import modal
from skimage.morphology import remove_small_objects
import scipy

## Selective Filters (do not necessarily apply to all pixels) ##

def lone_object_filter(image, min_size=2, connectivity=1, kernel_size=3):
    """
    Replaces isolated, contiguous regions of values in a raster with values
    representing the surrounding pixels.

    More specifically, this reduces noise in a raster by setting
    contiguous regions of values greater than a specified minimum size to
    the modal value in a specified neighborhood.

    The default argument values filter out lone, singular pixels.
    This filter is not idempotent, so it may need to be applied repeatedly
    until the output stops changing or the results are acceptable.

    Args:
        image (numpy.ndarray):
            The image to filter. Must not contain NaNs.
        min_size (int):
            Defines the minimum number of contiguous pixels that will not
            be set to the modal value of their neighborhood. Must be greater than 2.
            Setting this to 1 is pointless, since this function will then do nothing to the raster.
        connectivity (int):
            The maximum distance between any two pixels such that they are
            considered one group. For example, a connectivity of 1 considers
            only adjacent values to be within one group (contiguous areas), but 
            a connectivity of 2 also considers diagonally connected values to be 
            within one group. Must be greater than 0.
        kernel_size (int or float):
            The diameter of the circular kernel to use for the modal filter.
            If there are still pixels that should be set to the
            modal value of their neighborhood, increase this value to remove them.
            Note that the larger this value is, the more detail will tend to be
            lost in the image and the slower this function will run.

    Returns:
        The filtered image.

    Authors:
        <NAME> (<EMAIL>)\n
        <NAME>    (<EMAIL>)
    """
    assert kernel_size % 2 == 1, "The parameter `kernel_size` must be an odd number."
    image_min, image_max = image.min(), image.max()
    image_dtype = image.dtype
    image = np.interp(image, [image_min, image_max], [0,255]).astype(np.uint8)
    modal_filtered = modal(image, create_circular_mask(kernel_size, kernel_size))
    
    da = xr.DataArray(image)
    for i, val in enumerate(np.unique(image)):
        # Determine the pixels with this value that will be not be filtered (True to keep).
        layer = remove_small_objects(image == val, min_size=min_size, connectivity=connectivity)
        # Select the values from the image that will remain (filter it).
        filtered = da.where(layer) if i == 0 else filtered.combine_first(da.where(layer))
    # Fill in the removed values with their local modes.
{'fraction_non_alphanumeric': 0.05568108457069077, 'fraction_numerical': 0.005164622336991607, 'mean_word_length': 2.8755472170106318, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2998347', 'n_tokens_mistral': 1600, 'n_tokens_neox': 1531, 'n_words': 759}