<commit_before>from __future__ import division, absolute_import, print_function
from past.builtins import xrange

import fitsio
import numpy as np
import esutil
import copy
import sys

from .cluster import ClusterCatalog
from .background import Background
from .mask import HPMask
from .galaxy import GalaxyCatalog
from .cluster import Cluster
from .cluster import ClusterCatalog
from .depthmap import DepthMap
from .zlambda import Zlambda
from .zlambda import ZlambdaCorrectionPar
from .cluster_runner import ClusterRunner
from .centering import CenteringBCG, CenteringWcenZred, CenteringRandom, CenteringRandomSatellite

###################################################
# Order of operations:
#  __init__()
#    _additional_initialization() [override this]
#  run()
#    _setup()
#    _more_setup() [override this]
#    _process_cluster() [override this]
#    _postprocess() [override this]
#  output()
###################################################

class RunPercolation(ClusterRunner):
    """
    """

    def _additional_initialization(self, **kwargs):
        self.runmode = 'percolation'
        self.read_zreds = True
        self.zreds_required = True
        self.zredbkg_required = True
        self.filetype = 'final'

    def _more_setup(self, *args, **kwargs):

        print("%d: Percolation using catfile: %s" % (self.config.d.hpix, self.config.catfile))

        # read in the catalog...
        self.cat = ClusterCatalog.from_catfile(self.config.catfile,
                                               zredstr=self.zredstr,
                                               config=self.config,
                                               bkg=self.bkg,
                                               zredbkg=self.zredbkg,
                                               cosmo=self.cosmo,
                                               r0=self.r0,
                                               beta=self.beta)

        self.keepz = kwargs.pop('keepz', False)
        self.keepid = kwargs.pop('keepid', False)
        self.specseed = kwargs.pop('specseed', False)

        zrange=copy.copy(self.config.zrange)
        if self.keepz:
            zrange[0] -= self.config.calib_zrange_cushion
            zrange[0] = zrange[0] if zrange[0] > 0.05 else 0.05
            zrange[1] += self.config.calib_zrange_cushion

        use, = np.where((self.cat.z > zrange[0]) &
                        (self.cat.z < zrange[1]) &
                        (np.isfinite(self.cat.lnlike)) &
                        (self.cat.Lambda > self.config.percolation_minlambda))

        # How to bail if use.size == 0?  Need a framework for fail...

        if self.keepid:
            st = np.argsort(self.cat.mem_match_id[use])
        else:
            # Reverse sort by total likelihood
            st = np.argsort(self.cat.lnlike[use])[::-1]
            self.cat.mem_match_id[:] = 0

        self.cat = self.cat[use[st]]

        # This preserves previously set ids
        self._generate_mem_match_ids()

        self.cat.ra_orig = self.cat.ra
        self.cat.dec_orig = self.cat.dec

        self.do_percolation_masking = True
        self.do_lam_plusminus = True
        self.record_members = True
        self.do_correct_zlambda = True
        self.do_pz = True

        self.use_memradius = False
        if self.config.percolation_memradius > 1.0:
            self.use_memradius = True

        self.use_memlum = False
        if (self.config.lval_reference > 0.1):
            self.limlum = self.limlum if self.limlum > 0.1 else 0.1

        if (self.config.percolation_memlum > 0.0 and
            self.config.percolation_memlum < self.config.lval_reference):
            if self.config.percolation_memlum < self.limlum:
                self.limlum = self.config.percolation_memlum

        if self.config.percolation_lmask > 0.0:<commit_msg>Only check percolation_memradius and memlum if they are not None<commit_after>from __future__ import division, absolute_import, print_function
from past.builtins import xrange
from functools import reduce

import fitsio
import numpy as np
import esutil
import copy
import sys

from .cluster import ClusterCatalog
from .background import Background
from .mask import HPMask
from .galaxy import GalaxyCatalog
from .cluster import Cluster
from .cluster import ClusterCatalog
from .depthmap import DepthMap
from .zlambda import Zlambda
from .zlambda import ZlambdaCorrectionPar
from .cluster_runner import ClusterRunner
from .centering import CenteringBCG, CenteringWcenZred, CenteringRandom, CenteringRandomSatellite

###################################################
# Order of operations:
#  __init__()
#    _additional_initialization() [override this]
#  run()
#    _setup()
#    _more_setup() [override this]
#    _process_cluster() [override this]
#    _postprocess() [override this]
#  output()
###################################################

class RunPercolation(ClusterRunner):
    """
    """

    def _additional_initialization(self, **kwargs):
        self.runmode = 'percolation'
        self.read_zreds = True
        self.zreds_required = True
        self.zredbkg_required = True
        self.filetype = 'final'

    def _more_setup(self, *args, **kwargs):

        print("%d: Percolation using catfile: %s" % (self.config.d.hpix, self.config.catfile))

        # read in the catalog...
        self.cat = ClusterCatalog.from_catfile(self.config.catfile,
                                               zredstr=self.zredstr,
                                               config=self.config,
                                               bkg=self.bkg,
                                               zredbkg=self.zredbkg,
                                               cosmo=self.cosmo,
                                               r0=self.r0,
                                               beta=self.beta)

        self.keepz = kwargs.pop('keepz', False)
        self.keepid = kwargs.pop('keepid', False)
        self.specseed = kwargs.pop('specseed', False)

        zrange=copy.copy(self.config.zrange)
        if self.keepz:
            zrange[0] -= self.config.calib_zrange_cushion
            zrange[0] = zrange[0] if zrange[0] > 0.05 else 0.05
            zrange[1] += self.config.calib_zrange_cushion

        use, = np.where((self.cat.z > zrange[0]) &
                        (self.cat.z < zrange[1]) &
                        (np.isfinite(self.cat.lnlike)) &
                        (self.cat.Lambda > self.config.percolation_minlambda))

        # How to bail if use.size == 0?  Need a framework for fail...

        if self.keepid:
            st = np.argsort(self.cat.mem_match_id[use])
        else:
            # Reverse sort by total likelihood
            st = np.argsort(self.cat.lnlike[use])[::-1]
            self.cat.mem_match_id[:] = 0

        self.cat = self.cat[use[st]]

        # This preserves previously set ids
        self._generate_mem_match_ids()

        self.cat.ra_orig = self.cat.ra
        self.cat.dec_orig = self.cat.dec

        self.do_percolation_masking = True
        self.do_lam_plusminus = True
        self.record_members = True
        self.do_correct_zlambda = True
        self.do_pz = True

        self.use_memradius = False
        if (self.config.percolation_memradius is not None and
            self.config.percolation_memradius > 1.0):
            self.use_memradius = True

        self.use_memlum = False
        if (self.config.lval_reference > 0.1):
            self.limlum = self.limlum if self.limlum > 0.1 else 0.1

        if (self.config.percolation_memlum is not None and
            self.config.percolation_memlum > 0.0 and
            self.config.percolation_memlum < self.config.lval_reference):
            if self.config.percolation_memlum < self.limlum:
                self.limlum = self.config.percolation_memlum

        if self.config.percolation_lmask > 0.0:
{'fraction_non_alphanumeric': 0.11528251801289344, 'fraction_numerical': 0.0075843761850587785, 'mean_word_length': 1.9992418498862774, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5804175', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2274, 'n_words': 498}