<commit_before>import ee

from ..image_operation import ImageOperation


def mask_clouds(mosaic_def, collection):
    if not mosaic_def.mask_clouds:
        reduced = collection.select('cloud') \
            .reduce(ee.Reducer.sum()
                    .combine(ee.Reducer.count(), "", True)
                    .combine(ee.Reducer.min(), "", True))
        # Proportion of pixels that are cloudy
        cloud_proportion = reduced.select('cloud_sum') \
            .divide(reduced.select('cloud_count'))
        # A representative proportion of pixels that are cloudy cloudy for the neighborhood
        normal_cloud_proportion = cloud_proportion.reproject(crs='EPSG:4326', scale=10000) \
            .max(cloud_proportion.reproject(crs='EPSG:4326', scale=20000))
        # Measure of how a locations cloud proportion differs from the general area
        cloud_proportion_diff = cloud_proportion.subtract(normal_cloud_proportion)
        only_clouds = reduced.select('cloud_min')

        # When there is higher proportion of clouds than the normaly, keep the clouds.
        # It's probably something (typically buildings) misclassified as clouds.
        # Also, don't trust the cloud classification enough to completely mask area with only clouds
        # Desert sand can be classified as cloud.
        keep_clouds = cloud_proportion_diff.gt(0.4).And(normal_cloud_proportion.lt(0.3))
        keep_clouds = keep_clouds.Or(only_clouds)
    else:
        keep_clouds = False

    return collection.map(lambda image: _MaskClouds(image, mosaic_def).apply(keep_clouds))


class _MaskClouds(ImageOperation):
    def __init__(self, image, mosaic_def):
        super(_MaskClouds, self).__init__(image)
        self.mosaic_def = mosaic_def

    def apply(self, keep_clouds):
        cloud_free = buffer_mask(self.toImage('!i.cloud'), 300)
        # cloud_free = self.toImage('!i.cloud')
        to_mask = self.image.select('toMask')
        if keep_clouds:
            mask = to_mask.Not().And(cloud_free.Or(keep_clouds))
        else:
            mask = to_mask.Not().And(cloud_free)
        return self.image.updateMask(mask)
<commit_msg>Included filtered clumps of cloudy pixels.<commit_after>import ee

from ..image_operation import ImageOperation


def mask_clouds(mosaic_def, collection):
    if not mosaic_def.mask_clouds:
        reduced = collection.select('cloud') \
            .reduce(ee.Reducer.sum()
                    .combine(ee.Reducer.count(), "", True)
                    .combine(ee.Reducer.min(), "", True))
        # Proportion of pixels that are cloudy
        cloud_proportion = reduced.select('cloud_sum') \
            .divide(reduced.select('cloud_count'))
        # A representative proportion of pixels that are cloudy cloudy for the neighborhood
        normal_cloud_proportion = cloud_proportion.reproject(crs='EPSG:4326', scale=10000) \
            .max(cloud_proportion.reproject(crs='EPSG:4326', scale=20000))
        # Measure of how a locations cloud proportion differs from the general area
        cloud_proportion_diff = cloud_proportion.subtract(normal_cloud_proportion)
        only_clouds = reduced.select('cloud_min')

        # When there is higher proportion of clouds than the normaly, keep the clouds.
        # It's probably something (typically buildings) misclassified as clouds.
        # Also, don't trust the cloud classification enough to completely mask area with only clouds
        # Desert sand can be classified as cloud.
        keep_clouds = cloud_proportion_diff.gt(0.4).And(normal_cloud_proportion.lt(0.3))
        keep_clouds = keep_clouds.Or(only_clouds)
    else:
        keep_clouds = False

    return collection.map(lambda image: _MaskClouds(image, mosaic_def).apply(keep_clouds))


class _MaskClouds(ImageOperation):
    def __init__(self, image, mosaic_def):
        super(_MaskClouds, self).__init__(image)
        self.mosaic_def = mosaic_def

    def apply(self, keep_clouds):
        cloud_free = self.toImage('!i.cloud')
        cloud_free = buffer_mask(self.toImage('!i.cloud'), 300).And(cloud_free)
        to_mask = self.image.select('toMask')
        if keep_clouds:
            mask = to_mask.Not().And(cloud_free.Or(keep_clouds))
        else:
            mask = to_mask.Not().And(cloud_free)
        return self.image.updateMask(mask)

{'fraction_non_alphanumeric': 0.0975214269168404, 'fraction_numerical': 0.011582117211026175, 'mean_word_length': 3.0242311276794034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6347012', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1229, 'n_words': 293}