<commit_before>"""Class to consolidate distributed redmapper run.
"""
import os
import numpy as np
import glob
import fitsio
import re
import healpy as hp
import esutil

from ..configuration import Configuration
from ..volumelimit import VolumeLimitMask, VolumeLimitMaskFixed
from ..catalog import Catalog, Entry
from ..utilities import read_members, astro_to_sphere
from ..plotting import SpecPlot, NzPlot
from ..galaxy import GalaxyCatalog

class RedmapperConsolidateTask(object):
    """
    Class to consolidate a distributed redmapper run.

    This class looks for files of the specific format in the specified
    directory and consolidates, while applying richness and volume-limited
    cuts.
    """

    def __init__(self, configfile, lambda_cuts=None, vlim_lstars=[], path=None):
        """
        Instantiate a RedmapperConsolidateTask.

        Parameters
        ----------
        configfile: `str`
           Configuration yaml file.
        lambda_cuts: `list`, optional
           Richness cuts to apply.  Default is None.  If None,
           use the values in self.config.consolidate_lambda_cuts
        vlim_lstars: `list`, optional
           Volume-limit luminosity cuts to apply.  Default is [].
           If [] then use the values in self.config.consolidate_vlim_lstars.
        path: `str`, optional
           Path to look for files.  Default is config file path.
        """
        if path is None:
            outpath = os.path.dirname(os.path.abspath(configfile))
        else:
            outpath = path

        self.config = Configuration(configfile, outpath=path)

        if lambda_cuts is None:
            self.lambda_cuts = self.config.consolidate_lambda_cuts
        else:
            self.lambda_cuts = lambda_cuts

        if len(vlim_lstars) == 0:
            if len(self.config.consolidate_vlim_lstars) == 0:
                self.vlim_lstars = []
            else:
                self.vlim_lstars = self.config.consolidate_vlim_lstars
        else:
            self.vlim_lstars = vlim_lstars

    def run(self, do_plots=True, match_spec=True):
        """
        Run the redmapper consolidation task.

        Parameters
        ----------
        do_plots: `bool`, optional
           Make diagnostic plots.  Default is True.
        match_spec: `bool`, optional
           Match cluster centrals and members to spectra.
           Default is True.
        """

        # find the files
        catfiles = sorted(glob.glob(os.path.join(self.config.outpath, '%s_*_?????_final.fit' % (self.config.outbase))))

        self.config.logger.info("Found %d catalog files in %s" % (len(catfiles), self.config.outpath))

        # Extract the nside that was run
        m = re.search('_(\d+)_(\d\d\d\d\d)_', catfiles[0])
        if m is None:
            raise RuntimeError("Could not understand filename for %s" % (catfiles[0]))

        nside = int(m.groups()[0])

        if match_spec and not self.config.has_truth:<commit_msg>Fix bug in file search for consolidate code<commit_after>"""Class to consolidate distributed redmapper run.
"""
import os
import numpy as np
import glob
import fitsio
import re
import healpy as hp
import esutil

from ..configuration import Configuration
from ..volumelimit import VolumeLimitMask, VolumeLimitMaskFixed
from ..catalog import Catalog, Entry
from ..utilities import read_members, astro_to_sphere
from ..plotting import SpecPlot, NzPlot
from ..galaxy import GalaxyCatalog

class RedmapperConsolidateTask(object):
    """
    Class to consolidate a distributed redmapper run.

    This class looks for files of the specific format in the specified
    directory and consolidates, while applying richness and volume-limited
    cuts.
    """

    def __init__(self, configfile, lambda_cuts=None, vlim_lstars=[], path=None):
        """
        Instantiate a RedmapperConsolidateTask.

        Parameters
        ----------
        configfile: `str`
           Configuration yaml file.
        lambda_cuts: `list`, optional
           Richness cuts to apply.  Default is None.  If None,
           use the values in self.config.consolidate_lambda_cuts
        vlim_lstars: `list`, optional
           Volume-limit luminosity cuts to apply.  Default is [].
           If [] then use the values in self.config.consolidate_vlim_lstars.
        path: `str`, optional
           Path to look for files.  Default is config file path.
        """
        if path is None:
            outpath = os.path.dirname(os.path.abspath(configfile))
        else:
            outpath = path

        self.config = Configuration(configfile, outpath=path)

        if lambda_cuts is None:
            self.lambda_cuts = self.config.consolidate_lambda_cuts
        else:
            self.lambda_cuts = lambda_cuts

        if len(vlim_lstars) == 0:
            if len(self.config.consolidate_vlim_lstars) == 0:
                self.vlim_lstars = []
            else:
                self.vlim_lstars = self.config.consolidate_vlim_lstars
        else:
            self.vlim_lstars = vlim_lstars

    def run(self, do_plots=True, match_spec=True):
        """
        Run the redmapper consolidation task.

        Parameters
        ----------
        do_plots: `bool`, optional
           Make diagnostic plots.  Default is True.
        match_spec: `bool`, optional
           Match cluster centrals and members to spectra.
           Default is True.
        """

        # find the files
        catfiles = sorted(glob.glob(os.path.join(self.config.outpath, '%s_*_?????_final_catalog.fit' % (self.config.outbase))))

        self.config.logger.info("Found %d catalog files in %s" % (len(catfiles), self.config.outpath))

        # Extract the nside that was run
        m = re.search('_(\d+)_(\d\d\d\d\d)_', catfiles[0])
        if m is None:
            raise RuntimeError("Could not understand filename for %s" % (catfiles[0]))

        nside = int(m.groups()[0])

        if match_spec and not self.config.has_truth:
{'fraction_non_alphanumeric': 0.09224598930481283, 'fraction_numerical': 0.0016711229946524064, 'mean_word_length': 2.2993384785005513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5822745', 'n_tokens_mistral': 1741, 'n_tokens_neox': 1662, 'n_words': 512}