<commit_before>from __future__ import division, print_function

import numpy as np
import lsst.afw.image as afwImage
import lsst.afw.detection as afwDet
from astropy.table import Table, vstack
from astropy.convolution import Gaussian2DKernel
import photutils as phut
from . import utils
from . import imtools

__all__ = ['clean', 
           'find_blends',
           'image_threshold', 
           'measure_sources', 
           'photometry']


def clean(exposure, fpset_low, min_pix_low_thresh=100, name_high='THRESH_HIGH', 
          max_frac_high_thresh=0.3, rgrow=None):
    """
    Clean image of bright sources and associated diffuse regions by 
    replacing them with sky noise. Also, remove low-threshold regions 
    that are small. 

    Parameters
    ----------
    exposure : lsst.afw.image.ExposureF
        Exposure object with masks from hugsPipe.run.
    fpset_low : lsst.afw.detection.FootprintSet 
        Low threshold footprints.
    min_pix_low_thresh : int, optional
        Minimum number of pixels for a low-thresh footprint. 
    name_high : string, optional
        The name of the high-threshold bit plane.
    max_frac_high_thresh : float, optional
        Maximum fraction of high-thresh pixels to keep footprint. 
    rgrow : int, optional
        Number of pixels to grow footprints.

    Returns
    -------
    exp_clean : lsst.afw.ExposureF 
        The cleaned exposure object. 
    """
    
    # generate array of gaussian noise
    mi = exposure.getMaskedImage()
    mask = mi.getMask()
    shape = mask.getArray().shape
    back_rms = mi.getImage().getArray()[mask.getArray()==0].std()
    noise_array = back_rms*np.random.randn(shape[0], shape[1])

    # associate high thresh with low thresh and find small fps
    fpset_replace = afwDet.FootprintSet(mi.getBBox())
    fp_list = afwDet.FootprintList()
    for fp in fpset_low.getFootprints():
        hfp = afwDet.HeavyFootprintF(fp, mi)
        pix = hfp.getMaskArray()
        bits_hi = (pix & mask.getPlaneBitMask(name_high)!=0).sum()
        if bits_hi>0:
            ratio = bits_hi/float(fp.getArea())
            if ratio > max_frac_high_thresh:
                fp_list.append(fp)
        else:
            if fp.getArea() < min_pix_low_thresh:
                fp_list.append(fp)
    fpset_replace.setFootprints(fp_list)
    if rgrow:
        fpset_replace = afwDet.FootprintSet(fpset_replace, rgrow, True)
    mask.addMaskPlane('CLEANED')
    fpset_replace.setMask(mask, 'CLEANED')

    # create new exposure and replace footprints with noise
    exp_clean = exposure.clone()
    mi_clean = exp_clean.getMaskedImage()
    replace = mask.getArray() & mask.getPlaneBitMask('BRIGHT_OBJECT') != 0
    replace |= mask.getArray() & mask.getPlaneBitMask('CLEANED') != 0
    mi_clean.getImage().getArray()[replace] = noise_array[replace]
    return exp_clean


def find_blends(exp, fpset_det, name_low='THRESH_LOW', num_low_fps=2,
                min_fp_area=100, min_parent_area=0.3, min_parent_flux=0.8):
    """
    Search footprints for obvious blends of multiple small sources. 
    The search is carried out by counting the number of low-thresh 
    footprints within each detection footprint and looking at properties
    of the low-thresh footprints (e.g., size and flux).

    Parameters
    ----------
    exp : lsst.afw.image.ExposureF
        Exposure object. 
    fpset_det : lsst.afw.FootprintSet
        Detection footprint set.
    name_low : string, optional
        Name of the low-thresh bit plane.
    num_low_fps : int, optional
        Number of low-thresh fps to be considered blend.
    min_fp_area : int, optional
        Minimum number of pixels to be considered an object.
    min_parent_area : float, optional
        Minimum fraction of the footprint occupied by parent.
    min_parent_flux : float, optional
        Minimum fraction of the total flux that must 
        be contributed by the parent. 

    Notes
    -----
    A new bit plane called BLEND will be added to the mask 
    of the input exposure object, and DETECTED bits will be 
    turned off for blended sources. 
    """

    mask = exp.getMaskedImage().getMask()
    plane_low = mask.getPlaneBitMask(name_low)
    fp_list = afwDet.FootprintList()
    
    for fp_det in fpset_det.getFootprints():

        # get bbox of detection footprint
        bbox = fp_det.getBBox()
        exp_fp = exp.Factory(exp, bbox, afwImage.PARENT) 
        mask_fp = exp_fp.getMaskedImage().getMask()
        mi_fp = exp_fp.getMaskedImage()
        total_area = float(fp_det.getArea())

        # find low-thresh footprints in bbox
	fpset_low = afwDet.FootprintSet(
            mask_fp, afwDet.Threshold(plane_low, afwDet.Threshold.BITMASK))

        # find blends and save footprint
        if len(fpset_low.getFootprints()) >= num_low_fps:
            areas = np.array([fp_.getArea() for fp_ in
                              fpset_low.getFootprints()])
            biggest = areas.argmax()
            if (areas > min_fp_area).sum() >= num_low_fps:
                hfps = [afwDet.HeavyFootprintF(fp_, mi_fp)\
                        for fp_ in fpset_low.getFootprints()]
                fluxes = np.array(
                    [hfp_.getImageArray().sum()for hfp_ in hfps])
                is_blend = areas[biggest]/total_area < min_parent_area
                is_blend  |= fluxes[biggest]/fluxes.sum() < min_parent_flux
                if is_blend:
                    fp_list.append(fp_det)
                    mask_fp.clearMaskPlane(mask_fp.getMaskPlane('DETECTED'))
            elif (areas < min_fp_area).sum() == areas.size:
                fp_list.append(fp_det)
                mask_fp.clearMaskPlane(mask_fp.getMaskPlane('DETECTED'))
<commit_msg>indent error<commit_after>from __future__ import division, print_function

import numpy as np
import lsst.afw.image as afwImage
import lsst.afw.detection as afwDet
from astropy.table import Table, vstack
from astropy.convolution import Gaussian2DKernel
import photutils as phut
from . import utils
from . import imtools

__all__ = ['clean', 
           'find_blends',
           'image_threshold', 
           'measure_sources', 
           'photometry']


def clean(exposure, fpset_low, min_pix_low_thresh=100, name_high='THRESH_HIGH', 
          max_frac_high_thresh=0.3, rgrow=None):
    """
    Clean image of bright sources and associated diffuse regions by 
    replacing them with sky noise. Also, remove low-threshold regions 
    that are small. 

    Parameters
    ----------
    exposure : lsst.afw.image.ExposureF
        Exposure object with masks from hugsPipe.run.
    fpset_low : lsst.afw.detection.FootprintSet 
        Low threshold footprints.
    min_pix_low_thresh : int, optional
        Minimum number of pixels for a low-thresh footprint. 
    name_high : string, optional
        The name of the high-threshold bit plane.
    max_frac_high_thresh : float, optional
        Maximum fraction of high-thresh pixels to keep footprint. 
    rgrow : int, optional
        Number of pixels to grow footprints.

    Returns
    -------
    exp_clean : lsst.afw.ExposureF 
        The cleaned exposure object. 
    """
    
    # generate array of gaussian noise
    mi = exposure.getMaskedImage()
    mask = mi.getMask()
    shape = mask.getArray().shape
    back_rms = mi.getImage().getArray()[mask.getArray()==0].std()
    noise_array = back_rms*np.random.randn(shape[0], shape[1])

    # associate high thresh with low thresh and find small fps
    fpset_replace = afwDet.FootprintSet(mi.getBBox())
    fp_list = afwDet.FootprintList()
    for fp in fpset_low.getFootprints():
        hfp = afwDet.HeavyFootprintF(fp, mi)
        pix = hfp.getMaskArray()
        bits_hi = (pix & mask.getPlaneBitMask(name_high)!=0).sum()
        if bits_hi>0:
            ratio = bits_hi/float(fp.getArea())
            if ratio > max_frac_high_thresh:
                fp_list.append(fp)
        else:
            if fp.getArea() < min_pix_low_thresh:
                fp_list.append(fp)
    fpset_replace.setFootprints(fp_list)
    if rgrow:
        fpset_replace = afwDet.FootprintSet(fpset_replace, rgrow, True)
    mask.addMaskPlane('CLEANED')
    fpset_replace.setMask(mask, 'CLEANED')

    # create new exposure and replace footprints with noise
    exp_clean = exposure.clone()
    mi_clean = exp_clean.getMaskedImage()
    replace = mask.getArray() & mask.getPlaneBitMask('BRIGHT_OBJECT') != 0
    replace |= mask.getArray() & mask.getPlaneBitMask('CLEANED') != 0
    mi_clean.getImage().getArray()[replace] = noise_array[replace]
    return exp_clean


def find_blends(exp, fpset_det, name_low='THRESH_LOW', num_low_fps=2,
                min_fp_area=100, min_parent_area=0.3, min_parent_flux=0.8):
    """
    Search footprints for obvious blends of multiple small sources. 
    The search is carried out by counting the number of low-thresh 
    footprints within each detection footprint and looking at properties
    of the low-thresh footprints (e.g., size and flux).

    Parameters
    ----------
    exp : lsst.afw.image.ExposureF
        Exposure object. 
    fpset_det : lsst.afw.FootprintSet
        Detection footprint set.
    name_low : string, optional
        Name of the low-thresh bit plane.
    num_low_fps : int, optional
        Number of low-thresh fps to be considered blend.
    min_fp_area : int, optional
        Minimum number of pixels to be considered an object.
    min_parent_area : float, optional
        Minimum fraction of the footprint occupied by parent.
    min_parent_flux : float, optional
        Minimum fraction of the total flux that must 
        be contributed by the parent. 

    Notes
    -----
    A new bit plane called BLEND will be added to the mask 
    of the input exposure object, and DETECTED bits will be 
    turned off for blended sources. 
    """

    mask = exp.getMaskedImage().getMask()
    plane_low = mask.getPlaneBitMask(name_low)
    fp_list = afwDet.FootprintList()
    
    for fp_det in fpset_det.getFootprints():

        # get bbox of detection footprint
        bbox = fp_det.getBBox()
        exp_fp = exp.Factory(exp, bbox, afwImage.PARENT) 
        mask_fp = exp_fp.getMaskedImage().getMask()
        mi_fp = exp_fp.getMaskedImage()
        total_area = float(fp_det.getArea())

        # find low-thresh footprints in bbox
        fpset_low = afwDet.FootprintSet(
            mask_fp, afwDet.Threshold(plane_low, afwDet.Threshold.BITMASK))

        # find blends and save footprint
        if len(fpset_low.getFootprints()) >= num_low_fps:
            areas = np.array([fp_.getArea() for fp_ in
                              fpset_low.getFootprints()])
            biggest = areas.argmax()
            if (areas > min_fp_area).sum() >= num_low_fps:
                hfps = [afwDet.HeavyFootprintF(fp_, mi_fp)\
                        for fp_ in fpset_low.getFootprints()]
                fluxes = np.array(
                    [hfp_.getImageArray().sum()for hfp_ in hfps])
                is_blend = areas[biggest]/total_area < min_parent_area
                is_blend  |= fluxes[biggest]/fluxes.sum() < min_parent_flux
                if is_blend:
                    fp_list.append(fp_det)
                    mask_fp.clearMaskPlane(mask_fp.getMaskPlane('DETECTED'))
            elif (areas < min_fp_area).sum() == areas.size:
                fp_list.append(fp_det)
                mask_fp.clearMaskPlane(mask_fp.getMaskPlane('DETECTED'))

{'fraction_non_alphanumeric': 0.08917975567190227, 'fraction_numerical': 0.0036649214659685864, 'mean_word_length': 2.474083055471355, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1842034', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3537, 'n_words': 901}