<commit_before>"""
Utilities for ingesting Stack data products into the MySQL tables.
"""
from __future__ import absolute_import, print_function, division
import os
import sys
from collections import OrderedDict
import numpy as np
import sqlite3
import MySQLdb as Database
import astropy.io.fits as fits
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
import lsst.daf.persistence as dp
from .Pserv import DbConnection, create_csv_file_from_fits

def ingest_registry(connection, registry_file):
    """
    Ingest some relevant data from a registry.sqlite3 file into
    the CcdVisit table.
    """
    registry = sqlite3.connect(registry_file)
    query = """select taiObs, visit, filter, raft, ccd,
               expTime from raw where channel='0,0' order by visit asc"""
    for row in registry.execute(query):
        taiObs, visit, filter_, raft, ccd, expTime = tuple(row)
        taiObs = taiObs[:len('2016-03-18 00:00:00.000000')]
        query = """insert into CcdVisit set ccdVisitId=%(visit)i,
                   visitId=%(visit)i, ccdName='%(ccd)s',
                   raftName='%(raft)s', filterName='%(filter_)s',
                   obsStart='%(taiObs)s'
                   on duplicate key update
                   visitId=%(visit)i, ccdName='%(ccd)s',
                   raftName='%(raft)s', filterName='%(filter_)s',
                   obsStart='%(taiObs)s'""" % locals()
        try:
            connection.apply(query)
        except Database.DatabaseError as eobj:
            print("query:", query)
            raise eobj

def ingest_calexp_info(connection, repo):
    """
    Extract information such as seeing, sky background, sky noise,
    etc., from the calexp results and insert the values into the
    CcdVisit table.
    """
    # Use the Butler to find all of the visit/sensor combinations.
    butler = dp.Butler(repo)
    datarefs = butler.subset('calexp')
    for dataref in datarefs:
        calexp = dataref.get('calexp')
        calexp_bg = dataref.get('calexpBackground')

        # @todo Need to update this to use a value that takes into account
        # the raft and sensor info.
        ccdVisitId = dataref.dataId['visit']

        # Compute seeing, skyBg, skyNoise column values.
        pixel_scale = calexp.getWcs().pixelScale().asArcseconds()
        # For psf_fwhm (=seeing) calculation, see
        # https://github.com/lsst/meas_deblender/blob/master/python/lsst/meas/deblender/deblend.py#L227
        seeing = (calexp.getPsf().computeShape().getDeterminantRadius()
                  *2.35*pixel_scale)
        # Retrieving the nominal background image is computationally
        # expensive and just returns an interpolated version of the
        # stats_image (see
        # https://github.com/lsst/afw/blob/master/src/math/BackgroundMI.cc#L87).
        # So get the stats image instead.
        #bg_image = calexp_bg.getImage()
        bg_image = calexp_bg[0][0].getStatsImage()
        skyBg = afwMath.makeStatistics(bg_image, afwMath.MEDIAN).getValue()
        skyNoise = afwMath.makeStatistics(calexp.getMaskedImage(),
                                          afwMath.VARIANCECLIP).getValue()
        query = """update CcdVisit set seeing=%(seeing)e,<commit_msg>add a function to generate ccdVisitId<commit_after>"""
Utilities for ingesting Stack data products into the MySQL tables.
"""
from __future__ import absolute_import, print_function, division
import os
import sys
from collections import OrderedDict
import numpy as np
import sqlite3
import MySQLdb as Database
import astropy.io.fits as fits
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
import lsst.daf.persistence as dp
from .Pserv import DbConnection, create_csv_file_from_fits

def make_ccdVisitId(visit, raft, sensor):
    """
    Return an integer that uniquely identifies a visit-raft-sensor
    combination.
    """
    # @todo Include raft and sensor info into the return value.
    return visit

def ingest_registry(connection, registry_file):
    """
    Ingest some relevant data from a registry.sqlite3 file into
    the CcdVisit table.
    """
    registry = sqlite3.connect(registry_file)
    query = """select taiObs, visit, filter, raft, ccd,
               expTime from raw where channel='0,0' order by visit asc"""
    for row in registry.execute(query):
        taiObs, visit, filter_, raft, ccd, expTime = tuple(row)
        taiObs = taiObs[:len('2016-03-18 00:00:00.000000')]
        ccdVisitId = make_ccdVisitId(visit, raft, ccd)
        query = """insert into CcdVisit set ccdVisitId=%(ccdVisitId)i,
                   visitId=%(visit)i, ccdName='%(ccd)s',
                   raftName='%(raft)s', filterName='%(filter_)s',
                   obsStart='%(taiObs)s'
                   on duplicate key update
                   visitId=%(visit)i, ccdName='%(ccd)s',
                   raftName='%(raft)s', filterName='%(filter_)s',
                   obsStart='%(taiObs)s'""" % locals()
        try:
            connection.apply(query)
        except Database.DatabaseError as eobj:
            print("query:", query)
            raise eobj

def ingest_calexp_info(connection, repo):
    """
    Extract information such as seeing, sky background, sky noise,
    etc., from the calexp products and insert the values into the
    CcdVisit table.
    """
    # Use the Butler to find all of the visit/sensor combinations.
    butler = dp.Butler(repo)
    datarefs = butler.subset('calexp')
    for dataref in datarefs:
        calexp = dataref.get('calexp')
        calexp_bg = dataref.get('calexpBackground')
        ccdVisitId = make_ccdVisitId(dataref.dataId['visit'],
                                     dataref.dataId['raft'],
                                     dataref.dataId['sensor'])

        # Compute seeing, skyBg, skyNoise column values.
        pixel_scale = calexp.getWcs().pixelScale().asArcseconds()
        # For psf_fwhm (=seeing) calculation, see
        # https://github.com/lsst/meas_deblender/blob/master/python/lsst/meas/deblender/deblend.py#L227
        seeing = (calexp.getPsf().computeShape().getDeterminantRadius()
                  *2.35*pixel_scale)
        # Retrieving the nominal background image is computationally
        # expensive and just returns an interpolated version of the
        # stats_image (see
        # https://github.com/lsst/afw/blob/master/src/math/BackgroundMI.cc#L87),
        # so just get the stats image.
        #bg_image = calexp_bg.getImage()
        bg_image = calexp_bg[0][0].getStatsImage()
        skyBg = afwMath.makeStatistics(bg_image, afwMath.MEDIAN).getValue()
        skyNoise = afwMath.makeStatistics(calexp.getMaskedImage(),
                                          afwMath.VARIANCECLIP).getValue()
        query = """update CcdVisit set seeing=%(seeing)e,
{'fraction_non_alphanumeric': 0.10368629754736379, 'fraction_numerical': 0.010280511088265531, 'mean_word_length': 2.6514745308310994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2330247', 'n_tokens_mistral': 2151, 'n_tokens_neox': 1987, 'n_words': 533}