<commit_before>"""
Based upon examples by <NAME> (<EMAIL>) found here:
https://stash.lsstcorp.org/projects/SIM/repos/sims_catutils/browse/python/lsst/sims/
        catUtils/exampleCatalogDefinitions/phoSimCatalogExamples.py
"""

from lsst.sims.catalogs.measures.instance import CompoundInstanceCatalog
from lsst.sims.catalogs.generation.db import CatalogDBObject
from lsst.sims.catUtils.baseCatalogModels import OpSim3_61DBObject
from lsst.sims.catUtils.exampleCatalogDefinitions.phoSimCatalogExamples import \
        PhoSimCatalogPoint, PhoSimCatalogSersic2D, PhoSimCatalogZPoint
from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
#from sprinkler import sprinklerCompound

class InstcatGenerator(object):
    _starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars',
                     'cepheidstars']
    def __init__(self, opsim_db, fieldRA, fieldDec, boundLength=0.3):
        gen = ObservationMetaDataGenerator(database=opsim_db,
                                           driver='sqlite')
        self.obs_md_results = gen.getObservationMetaData(fieldRA=fieldRA, 
                                                         fieldDec=fieldDec,
                                                         boundLength=boundLength)

    def find_visits(self, bandpass, nmax=None):
        visits = []
        for obs_metadata in self.obs_md_results:
            if nmax is not None and len(visits) == nmax:
                break
            if obs_metadata.bandpass == bandpass:
                visits.append(obs_metadata)
        return visits

    def __call__(self, outfile, obs_metadata):
        catalogs = []

        # Add Instance Catalogs for phoSim stars.
        for starName in self._starObjNames:
            starDBObj = CatalogDBObject.from_objid(starName)
            catalogs.append(PhoSimCatalogPoint(starDBObj, 
                                               obs_metadata=obs_metadata))

        # Add phosim Galaxy Instance Catalogs to compound Instance Catalog.
        galsBulge = CatalogDBObject.from_objid('galaxyBulge')
        catalogs.append(PhoSimCatalogSersic2D(galsBulge,
                                              obs_metadata=obs_metadata))
        galsDisk = CatalogDBObject.from_objid('galaxyDisk')
        catalogs.append(PhoSimCatalogSersic2D(galsDisk,
                                              obs_metadata=obs_metadata))
        galsAGN = CatalogDBObject.from_objid('galaxyAgn')
        catalogs.append(PhoSimCatalogZPoint(galsAGN,
                                            obs_metadata=obs_metadata))

        # Write the catalogs to the output file one at a time.
        write_header = True
        for catalog in catalogs:
            catalog.write_catalog(outfile, write_mode='a',
                                  write_header=write_header,
                                  chunk_size=20000)
            write_header = False

if __name__ == '__main__':
    import os
    import pickle
    import time

    # This following is a deep drilling field ID for enigma_1189, but
    # fieldID is not one of the selection options in
    # getObservationMetaData(...), so we need to continue using
    # fieldRA, fieldDec
    fieldID = 1427
    
    fieldRA = (53, 54)
    fieldDec = (-29, -27)

    opsim_db = '/nfs/slac/g/ki/ki18/jchiang/DESC/Twinkles/work/enigma_1189_sqlite.db'
    pickle_file = 'instcat_generator_enigma_1189_%(fieldID)i.pickle' % locals()

    t0 = time.time()
    if not os.path.isfile(pickle_file):
        print "Extracting visits from %s:" % os.path.basename(opsim_db)
        generator = InstcatGenerator(opsim_db, fieldRA, fieldDec)
        pickle.dump(generator, open(pickle_file, 'w'))
        print "execution time:", time.time() - t0
    else:
        print "Loading pickle file with visits:", pickle_file
        generator = pickle.load(open(pickle_file))
        print "execution time:", time.time() - t0

    nmax = 1
    for bandpass in 'ugrizy':
        print "band pass:", bandpass
        visits = generator.find_visits(bandpass, nmax=nmax)
        for visit in visits:
            obshistid = visit.phoSimMetaData['Opsim_obshistid'][0]
            outfile = 'phosim_input_%s_%07i.txt' % (bandpass, obshistid)
            print outfile
            generator(outfile, visit)<commit_msg>implement looping over try/except block to workaround intermittent connectivity to UW server<commit_after>"""
Based upon examples by <NAME> (<EMAIL>) found here:
https://stash.lsstcorp.org/projects/SIM/repos/sims_catutils/browse/python/lsst/sims/
        catUtils/exampleCatalogDefinitions/phoSimCatalogExamples.py
"""
import os
from lsst.sims.catalogs.generation.db import CatalogDBObject
from lsst.sims.catUtils.baseCatalogModels import OpSim3_61DBObject
from lsst.sims.catUtils.exampleCatalogDefinitions.phoSimCatalogExamples import \
        PhoSimCatalogPoint, PhoSimCatalogSersic2D, PhoSimCatalogZPoint
from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
#from sprinkler import sprinklerCompound

class InstcatGenerator(object):
    def __init__(self, opsim_db, fieldRA, fieldDec, boundLength=0.3):
        starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars',
                        'cepheidstars']
        self._phosimCatDefs = dict([(objid, PhoSimCatalogPoint) for objid 
                                    in starObjNames])
        self._phosimCatDefs.update(dict(galaxyBulge=PhoSimCatalogSersic2D,
                                        galaxyDisk=PhoSimCatalogSersic2D,
                                        galaxyAgn=PhoSimCatalogZPoint))
        gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite')
        self.obs_md_results = gen.getObservationMetaData(fieldRA=fieldRA, 
                                                         fieldDec=fieldDec,
                                                         boundLength=boundLength)
    def find_visits(self, bandpass, nmax=None):
        visits = []
        for obs_metadata in self.obs_md_results:
            if nmax is not None and len(visits) == nmax:
                break
            if obs_metadata.bandpass == bandpass:
                visits.append(obs_metadata)
        return visits
    def _processDbObject(self, objid, outfile, obs_md, write_header=False):
        while True:
            try:
                db_obj = CatalogDBObject.from_objid(objid)
                inst_cat = self._phosimCatDefs[objid](db_obj,
                                                      obs_metadata=obs_md)
                break
            except RuntimeError:
                continue
        inst_cat.write_catalog(outfile, write_mode='a',
                               write_header=write_header, chunk_size=20000)
    def write_catalog(self, outfile, obs_metadata, clobber=True):
        if clobber and os.path.isfile(outfile):
            os.remove(outfile)
        write_header = True
        for objid in self._phosimCatDefs:
            self._processDbObject(objid, outfile, obs_metadata,
                                  write_header=write_header)
            write_header = False

if __name__ == '__main__':
    import os
    import pickle
    import time

    fieldID = 1427
    fieldRA = (53, 54)
    fieldDec = (-29, -27)

    opsim_db = '/nfs/slac/g/ki/ki18/jchiang/DESC/Twinkles/work/enigma_1189_sqlite.db'
    pickle_file = 'instcat_generator_enigma_1189_%(fieldID)i.pickle' % locals()

    t0 = time.time()
    if not os.path.isfile(pickle_file):
        print "Extracting visits from %s:" % os.path.basename(opsim_db)
        generator = InstcatGenerator(opsim_db, fieldRA, fieldDec)
        pickle.dump(generator, open(pickle_file, 'w'))
        print "execution time:", time.time() - t0
    else:
        print "Loading pickle file with visits:", pickle_file
        generator = pickle.load(open(pickle_file))
        print "execution time:", time.time() - t0

    nmax = 20
    for bandpass in 'ugrizy':
        print "band pass:", bandpass
        visits = generator.find_visits(bandpass, nmax=nmax)
        for i, visit in enumerate(visits):
            obshistid = visit.phoSimMetaData['Opsim_obshistid'][0]
            outfile = 'phosim_input_%s_%07i.txt' % (bandpass, obshistid)
            print i, outfile
            generator.write_catalog(outfile, visit)
{'fraction_non_alphanumeric': 0.08379618449649844, 'fraction_numerical': 0.010746196570876599, 'mean_word_length': 2.2713270142180093, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947960', 'n_tokens_mistral': 2507, 'n_tokens_neox': 2413, 'n_words': 494}