<commit_before>import nbformat as nbf
import re

DEFAULT_FLAGS = ['calib_psfUsed', 'qaBad_flag',
                 'merge_measurement_i',
                 'merge_measurement_r',
                 'merge_measurement_z',
                 'merge_measurement_y',
                 'merge_measurement_g',
                 'base_Footprint_nPix_flag',
                 'base_PixelFlags_flag_inexact_psfCenter']

DEFAULT_FUNCTORS = (('cmodel', "MagDiff('modelfit_CModel', 'base_PsfFlux')"),
                ('gauss', "MagDiff('base_GaussianFlux', 'base_PsfFlux')"),
                ('count', "Column('base_InputCount_value')"),
                ('seeing', "Seeing()"))

DEFAULT_MATCHED_FUNCTORS = (('gauss', "MagDiff('base_GaussianFlux', 'base_PsfFlux')"),
                            ('seeing', "Seeing()"))


DEFAULT_COLOR_FUNCTORS = (('psfmag', "Mag('base_PsfFlux')"),
                          ('cmodel', "Mag('modelfit_CModel')"),
                          ('kron',  "Mag('ext_photometryKron_KronFlux')"))

DEFAULT_COLOR_FLAGS = ['calib_psfUsed', 'qaBad_flag', 'base_PixelFlags_flag_inexact_psfCenter']

HSC_FILTERS = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
HSC_SHORT_FILTERS = 'GRIZY'
REFERENCE_FILTER = 'HSC-I'


class Cell(object):
    params = []
    code = ''
    def __call__(self, **kwargs):
        for p in self.params:
            if p not in kwargs:
                raise ValueError('Must provide {}'.format(p))
        return nbf.v4.new_code_cell(self.code.format(**kwargs))

class CodeCell(Cell):
    def __init__(self, code):
        self.code = code


class HvImportCell(Cell):
    code = """\
import holoviews as hv
hv.notebook_extension('bokeh')"""

class ButlerInitCell(Cell):
    params = ('repo',)
    code = """\
from lsst.daf.persistence import Butler
butler = Butler('{repo}')"""

class DaskClientCell(Cell):
    def __init__(self, scheduler_file):
        self.scheduler_file = scheduler_file
        
    @property
    def code(self):
        code = 'from distributed import Client\n'
        code += 'client = Client(scheduler_file={})'.format(self.scheduler_file)
        return code
    
class DefineCoaddCatalogCell(Cell):
    params = ('tract', 'filt')
    code = """\
from explorer.catalog import CoaddCatalog, VisitCatalog, MultiMatchedCatalog
from explorer.utils import get_visits

tract = {tract}
filt = "{filt}"
dataId = {{'tract':tract, 'filter':filt}}
catalog = CoaddCatalog(butler, dataId)"""

class DefineMatchedCatalogCell(Cell):
    params = ('tract', 'filt')
    code = """\
from explorer.catalog import CoaddCatalog, VisitCatalog, MultiMatchedCatalog
from explorer.utils import get_visits

tract = {tract}
filt = "{filt}"
dataId = {{'tract':tract, 'filter':filt}}
coaddCat = CoaddCatalog(butler, dataId)
visitCats = [VisitCatalog(butler, {{'tract': tract, 'filter':filt, 'visit':v}}, name=v) for v in get_visits(butler, tract, filt)]
catalog = MultiMatchedCatalog(coaddCat, visitCats, match_registry='QAmatchRegistry.h5')"""
    
class DefineColorCatalogCell(Cell):
    params = ('tract', 'filters', 'short_filters', 'reference_filter')
    code = """\
from explorer.catalog import CoaddCatalog, MultiBandCatalog

tract = 9615
filts = {filters}
short_filts = '{short_filters}'
reference_filt = '{reference_filter}'

catalog = MultiBandCatalog({{filt: CoaddCatalog(butler, {{'tract':tract, 'filter':filt}}, name=filt) for filt in filts}}, 
                           short_filters=short_filts, reference_filt=reference_filt)"""


class DefineFunctorsCell(Cell):
        
    def __init__(self, functors=DEFAULT_FUNCTORS):
        self.functors = functors
        
    @property
    def import_command(self):
        to_import = set()
        for k, v in self.functors:
            m = re.search('(\w+)\(', v)
            to_import.add(m.group(1))
        return 'from explorer.functors import ' + ','.join(to_import) + '\n'
    
    @property
    def code(self):
        code = self.import_command
        for k, v in self.functors:
            code += '{} = {}\n'.format(k,v)
        
        code += '\nfuncs = {{'
        for k, v in self.functors:
            code += "'{0}':{0},".format(k)
        code += '}}'
        return code
    
    
class DefineDatasetCell(Cell):
    def __init__(self, flags=DEFAULT_FLAGS, client=False):
        self.flags = flags
        self.client = client
    <commit_msg>add comment<commit_after>import nbformat as nbf
import re

DEFAULT_FLAGS = ['calib_psfUsed', 'qaBad_flag',
                 'merge_measurement_i',
                 'merge_measurement_r',
                 'merge_measurement_z',
                 'merge_measurement_y',
                 'merge_measurement_g',
                 'base_Footprint_nPix_flag',
                 'base_PixelFlags_flag_inexact_psfCenter']

DEFAULT_FUNCTORS = (('cmodel', "MagDiff('modelfit_CModel', 'base_PsfFlux')"),
                ('gauss', "MagDiff('base_GaussianFlux', 'base_PsfFlux')"),
                ('count', "Column('base_InputCount_value')"),
                ('seeing', "Seeing()"))

DEFAULT_MATCHED_FUNCTORS = (('gauss', "MagDiff('base_GaussianFlux', 'base_PsfFlux')"),
                            ('seeing', "Seeing()"))


DEFAULT_COLOR_FUNCTORS = (('psfmag', "Mag('base_PsfFlux')"),
                          ('cmodel', "Mag('modelfit_CModel')"),
                          ('kron',  "Mag('ext_photometryKron_KronFlux')"))

DEFAULT_COLOR_FLAGS = ['calib_psfUsed', 'qaBad_flag', 'base_PixelFlags_flag_inexact_psfCenter']

HSC_FILTERS = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
HSC_SHORT_FILTERS = 'GRIZY'
REFERENCE_FILTER = 'HSC-I'


class Cell(object):
    params = []
    code = ''
    def __call__(self, **kwargs):
        for p in self.params:
            if p not in kwargs:
                raise ValueError('Must provide {}'.format(p))
        return nbf.v4.new_code_cell(self.code.format(**kwargs))

class CodeCell(Cell):
    def __init__(self, code):
        self.code = code


class HvImportCell(Cell):
    code = """\
import holoviews as hv
hv.notebook_extension('bokeh')"""

class ButlerInitCell(Cell):
    params = ('repo',)
    code = """\
from lsst.daf.persistence import Butler
butler = Butler('{repo}')"""

class DaskClientCell(Cell):
    def __init__(self, scheduler_file):
        self.scheduler_file = scheduler_file
        
    @property
    def code(self):
        code = 'from distributed import Client\n'
        code += 'client = Client(scheduler_file={})'.format(self.scheduler_file)
        return code
    
class DefineCoaddCatalogCell(Cell):
    params = ('tract', 'filt')
    code = """\
from explorer.catalog import CoaddCatalog, VisitCatalog, MultiMatchedCatalog
from explorer.utils import get_visits

tract = {tract}
filt = "{filt}"
dataId = {{'tract':tract, 'filter':filt}}
catalog = CoaddCatalog(butler, dataId)"""

class DefineMatchedCatalogCell(Cell):
    params = ('tract', 'filt')
    code = """\
from explorer.catalog import CoaddCatalog, VisitCatalog, MultiMatchedCatalog
from explorer.utils import get_visits

tract = {tract}
filt = "{filt}"
dataId = {{'tract':tract, 'filter':filt}}
coaddCat = CoaddCatalog(butler, dataId)
visitCats = [VisitCatalog(butler, {{'tract': tract, 'filter':filt, 'visit':v}}, name=v) for v in get_visits(butler, tract, filt)]
catalog = MultiMatchedCatalog(coaddCat, visitCats, match_registry='QAmatchRegistry.h5')"""
    
class DefineColorCatalogCell(Cell):
    params = ('tract', 'filters', 'short_filters', 'reference_filter')
    code = """\
from explorer.catalog import CoaddCatalog, MultiBandCatalog

tract = 9615
filts = {filters}
short_filts = '{short_filters}'
reference_filt = '{reference_filter}'

catalog = MultiBandCatalog({{filt: CoaddCatalog(butler, {{'tract':tract, 'filter':filt}}, name=filt) for filt in filts}}, 
                           short_filters=short_filts, reference_filt=reference_filt)"""


class DefineFunctorsCell(Cell):
        
    def __init__(self, functors=DEFAULT_FUNCTORS):
        self.functors = functors
        
    @property
    def import_command(self):
        to_import = set()
        for k, v in self.functors:
            m = re.search('(\w+)\(', v)
            to_import.add(m.group(1))
        return 'from explorer.functors import ' + ','.join(to_import) + '\n'
    
    @property
    def code(self):
        code = '# Modify this cell to calculate whatever functors you want\n\n'
        code += self.import_command
        for k, v in self.functors:
            code += '{} = {}\n'.format(k,v)
        
        code += '\nfuncs = {{'
        for k, v in self.functors:
            code += "'{0}':{0},".format(k)
        code += '}}'
        return code
    
    
class DefineDatasetCell(Cell):
    def __init__(self, flags=DEFAULT_FLAGS, client=False):
        self.flags = flags
        self.client = client
    
{'fraction_non_alphanumeric': 0.144582431354677, 'fraction_numerical': 0.0020508146291443544, 'mean_word_length': 3.026605504587156, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161597', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2902, 'n_words': 581}