<commit_before>"""
desispec.calibfinder
==============

Reading and selecting calibration data from $DESI_SPECTRO_CALIB using content of image headers
"""

import re
import os
import numpy as np
import yaml
import os.path
from desiutil.log import get_logger


def _parse_date_obs(value):
    '''
    converts DATE-OBS keywork to int
    with for instance DATE-OBS=2016-12-21T18:06:21.268371-05:00
    '''
    m = re.search(r'(\d+)-(\d+)-(\d+)T', value)
    Y,M,D=tuple(map(int, m.groups()))
    dateobs=int(Y*10000+M*100+D)
    return dateobs


def findcalibfile(headers,key,yaml_file=None) :
    """
    read and select calibration data file from $DESI_SPECTRO_CALIB using the keywords found in the headers

    Args:
        headers: list of fits headers, or list of dictionnaries
        
    Optional:
            yaml_file: path to a specific yaml file. By default, the code will
            automatically find the yaml file from the environment variable
            DESI_SPECTRO_CALIB and the CAMERA keyword in the headers
    
    Returns path to calibration file
    """
    cfinder = CalibFinder(headers,yaml_file)
    return cfinder.findfile(key)

class CalibFinder() :

    
    def __init__(self,headers,yaml_file=None) :
        """
        Class to read and select calibration data from $DESI_SPECTRO_CALIB using the keywords found in the headers
        
        Args:
            headers: list of fits headers, or list of dictionnaries
        
        Optional:
            yaml_file: path to a specific yaml file. By default, the code will
            automatically find the yaml file from the environment variable
            DESI_SPECTRO_CALIB and the CAMERA keyword in the headers

        """
        log = get_logger()
                    
        old_version = False
        
        # temporary backward compatibility
        if not "DESI_SPECTRO_CALIB" in os.environ :
            if "DESI_CCD_CALIBRATION_DATA" in os.environ :
                log.warning("Using deprecated DESI_CCD_CALIBRATION_DATA env. variable to find calibration data\nPlease switch to DESI_SPECTRO_CALIB a.s.a.p.")
                self.directory = os.environ["DESI_CCD_CALIBRATION_DATA"]
                old_version = True
            else :
                log.error("Need environment variable DESI_SPECTRO_CALIB")
                raise KeyError("Need environment variable DESI_SPECTRO_CALIB")
        else :
            self.directory = os.environ["DESI_SPECTRO_CALIB"]
        
        if len(headers)==0 :
            log.error("Need at least a header")
            raise RuntimeError("Need at least a header")
        header=dict() 
        for other_header in headers :
            for k in other_header :
                if k not in header :
                    header[k]=other_header[k]
        
        cameraid=header["CAMERA"].strip().lower()
        dateobs=_parse_date_obs(header["DATE-OBS"])
        detector=header["DETECTOR"].strip()
        if "DOSVER" in header :
            dosver = header["DOSVER"].strip()
        else :
            dosver = None
        if "FEEVER" in header :
            feever = header["FEEVER"].strip()
        else :
            feever = None
        
        spectro=int(cameraid[-1])
        if yaml_file is None :
            if old_version :
                yaml_file = os.path.join(self.directory,"ccd_calibration.yaml")
            else :
                yaml_file = "{}/spec/sp{}/{}.yaml".format(self.directory,spectro,cameraid)<commit_msg>support finding SIM data with non-SIM $DESI_SPECTRO_CALIB<commit_after>"""
desispec.calibfinder
==============

Reading and selecting calibration data from $DESI_SPECTRO_CALIB using content of image headers
"""

import re
import os
import numpy as np
import yaml
import os.path
from desiutil.log import get_logger


def _parse_date_obs(value):
    '''
    converts DATE-OBS keywork to int
    with for instance DATE-OBS=2016-12-21T18:06:21.268371-05:00
    '''
    m = re.search(r'(\d+)-(\d+)-(\d+)T', value)
    Y,M,D=tuple(map(int, m.groups()))
    dateobs=int(Y*10000+M*100+D)
    return dateobs


def findcalibfile(headers,key,yaml_file=None) :
    """
    read and select calibration data file from $DESI_SPECTRO_CALIB using the keywords found in the headers

    Args:
        headers: list of fits headers, or list of dictionnaries
        
    Optional:
            yaml_file: path to a specific yaml file. By default, the code will
            automatically find the yaml file from the environment variable
            DESI_SPECTRO_CALIB and the CAMERA keyword in the headers
    
    Returns path to calibration file
    """
    cfinder = CalibFinder(headers,yaml_file)
    return cfinder.findfile(key)

class CalibFinder() :

    
    def __init__(self,headers,yaml_file=None) :
        """
        Class to read and select calibration data from $DESI_SPECTRO_CALIB using the keywords found in the headers
        
        Args:
            headers: list of fits headers, or list of dictionnaries
        
        Optional:
            yaml_file: path to a specific yaml file. By default, the code will
            automatically find the yaml file from the environment variable
            DESI_SPECTRO_CALIB and the CAMERA keyword in the headers

        """
        log = get_logger()
                    
        old_version = False
        
        # temporary backward compatibility
        if not "DESI_SPECTRO_CALIB" in os.environ :
            if "DESI_CCD_CALIBRATION_DATA" in os.environ :
                log.warning("Using deprecated DESI_CCD_CALIBRATION_DATA env. variable to find calibration data\nPlease switch to DESI_SPECTRO_CALIB a.s.a.p.")
                self.directory = os.environ["DESI_CCD_CALIBRATION_DATA"]
                old_version = True
            else :
                log.error("Need environment variable DESI_SPECTRO_CALIB")
                raise KeyError("Need environment variable DESI_SPECTRO_CALIB")
        else :
            self.directory = os.environ["DESI_SPECTRO_CALIB"]
        
        if len(headers)==0 :
            log.error("Need at least a header")
            raise RuntimeError("Need at least a header")
        header=dict() 
        for other_header in headers :
            for k in other_header :
                if k not in header :
                    header[k]=other_header[k]
        
        cameraid=header["CAMERA"].strip().lower()
        dateobs=_parse_date_obs(header["DATE-OBS"])
        detector=header["DETECTOR"].strip()
        if "DOSVER" in header :
            dosver = header["DOSVER"].strip()
        else :
            dosver = None
        if "FEEVER" in header :
            feever = header["FEEVER"].strip()
        else :
            feever = None

        # Support simulated data even if $DESI_SPECTRO_CALIB points to
        # real data calibrations
        self.directory = os.path.normpath(self.directory)  # strip trailing /
        if detector == "SIM" and (not self.directory.endswith("sim")):
            self.directory = os.path.join(self.directory, "sim")

        if not os.path.isdir(self.directory):
            raise IOError("Calibration directory {} not found".format(self.directory))

        spectro=int(cameraid[-1])
        if yaml_file is None :
            if old_version :
                yaml_file = os.path.join(self.directory,"ccd_calibration.yaml")
            else :
                yaml_file = "{}/spec/sp{}/{}.yaml".format(self.directory,spectro,cameraid)
{'fraction_non_alphanumeric': 0.08769580934529388, 'fraction_numerical': 0.009104297764091579, 'mean_word_length': 2.201885983711959, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1849356', 'n_tokens_mistral': 2250, 'n_tokens_neox': 2139, 'n_words': 601}