<commit_before>#!/usr/bin/env python
# coding: utf-8

import os
import glob
import numpy as np
from astropy.table import Table
from astropy.io import fits
## Import some helper functions, you can see their definitions by uncomenting the bash shell command
from desispec.workflow.utils import define_variable_from_environment, get_json_dict
from desispec.workflow.desi_proc_funcs import load_raw_data_header, cameras_from_raw_data
from desiutil.log import get_logger
from desispec.util import header2night
from desispec.io.util import create_camword, parse_badamps

#############################################
##### Exposure Table Column Definitions #####
#############################################
## To eventually being turned into a full-fledged data model. For now a brief description.
# EXPID, int, the exposure ID.
# EXPTIME, float, the exposure time.
# OBSTYPE, string, the obstype as defined by ICS.
# CAMWORD, string, typically 'a'+ str(spectrographs) defined by ICS unless specified by command line argument
# TILEID, int, the TILEID of the tile the exposure observed.
# TARGTRA, float, The TARGTRA as given by ICS. The RA of the target.
# TARGTDEC, float, The TARGTDEC as given by ICS. The DEC of the target.
# NIGHT, int, the night of the observation.
# PURPOSE, str, The purpose of the exposure as defined by ICS.
# FA_SURV, str, The survey designated/used by fiberassign
# SEQNUM, int, The number of the current exposure in a sequence of exposures, as given by ICS. If not a sequence, SEQNUM is 1.
# SEQTOT, int, The total number of exposures taken in the current sequence, as given by ICS. If not a sequence, SEQTOT is 1.
# PROGRAM, string, The program as given by ICS.
# MJD-OBS, float, The MJD-OBS as given by ICS. Modified Julian Date of the observation.
# BADCAMWORD, string, camword defining the bad cameras that should not be processed.
# BADAMPS, string, comma separated list of "{camera}{petal}{amp}", i.e. "[brz][0-9][ABCD]". Example: 'b7D,z8A'
#                  in the csv this is saved as a semicolon separated list
# LASTSTEP, string, the last step the pipeline should run through for the given exposure. Inclusive of last step.
# EXPFLAG, np.ndarray, set of flags that describe that describe the exposure.
# HEADERERR, np.ndarray, In the csv given as a "|" separated list of key=value pairs describing columns in the table that should
#                        be corrected. The workflow transforms these into an array of strings.
#                        NOTE: This will be used to change the given key/value pairs in the production table.
# COMMENTS, np.ndarray, In the csv given as either a "|" separated list of comments or one long comment. When loaded it
#                       is a numpy array of the strings.These are not used by the workflow but useful for humans
#                       to put notes for other humans.
##################################################

def exposure_table_column_defs():
    """
    Contains the column names, data types, and default row values for a DESI Exposure table. It returns
    the names, datatypes, and defaults as a list of 3-tuples.

    Args:
        None.

    Returns:
        columns, list. List of tuples (name, type, default).
    """
    columns = [
                ('EXPID', int, -99),
                ('OBSTYPE', 'S8', 'unknown'),
                ('TILEID', int, -99),
                ('LASTSTEP', 'S30', 'all'),
                ('CAMWORD', 'S30', 'a0123456789'),
                ('BADCAMWORD', 'S30', ''),
                ('BADAMPS', 'S30', ''),
                ('EXPTIME', float, 0.0),
                ('EFFTIME_ETC', float, -99.),
                ('SURVEY', 'S10', 'unknown'),
                ('FA_SURV', 'S10', 'unknown'),
                ('FAPRGRM', 'S10', 'unknown'),
                ('GOALTIME', float, -99.),
                ('GOALTYPE', 'S10', 'unknown'),
                ('EBVFAC', float, 1.0),
                ('AIRMASS', float, 1.0),
                ('SPEED', float, -99.0),
                ('TARGTRA', float, 89.99),
                ('TARGTDEC', float, -89.99),
                ('SEQNUM', int, 1),
                ('SEQTOT', int, 1),
                ('PROGRAM', 'S60', 'unknown'),
                ('PURPOSE', 'S30', 'unknown'),
                ('MJD-OBS', float, 50000.0),
                ('NIGHT', int, 20000101),
                ('HEADERERR', np.ndarray, np.array([], dtype=str)),
                ('EXPFLAG', np.ndarray, np.array([], dtype=str)),
                ('COMMENTS', np.ndarray, np.array([], dtype=str))
              ]
    return columns

def default_obstypes_for_exptable():
    """
    Defines the exposure types to be recognized by the workflow and saved in the exposure table by default.
<commit_msg>Increase exptab FAPRGRM char limit to 20<commit_after>#!/usr/bin/env python
# coding: utf-8

import os
import glob
import numpy as np
from astropy.table import Table
from astropy.io import fits
## Import some helper functions, you can see their definitions by uncomenting the bash shell command
from desispec.workflow.utils import define_variable_from_environment, get_json_dict
from desispec.workflow.desi_proc_funcs import load_raw_data_header, cameras_from_raw_data
from desiutil.log import get_logger
from desispec.util import header2night
from desispec.io.util import create_camword, parse_badamps

#############################################
##### Exposure Table Column Definitions #####
#############################################
## To eventually being turned into a full-fledged data model. For now a brief description.
# EXPID, int, the exposure ID.
# EXPTIME, float, the exposure time.
# OBSTYPE, string, the obstype as defined by ICS.
# CAMWORD, string, typically 'a'+ str(spectrographs) defined by ICS unless specified by command line argument
# TILEID, int, the TILEID of the tile the exposure observed.
# TARGTRA, float, The TARGTRA as given by ICS. The RA of the target.
# TARGTDEC, float, The TARGTDEC as given by ICS. The DEC of the target.
# NIGHT, int, the night of the observation.
# PURPOSE, str, The purpose of the exposure as defined by ICS.
# FA_SURV, str, The survey designated/used by fiberassign
# SEQNUM, int, The number of the current exposure in a sequence of exposures, as given by ICS. If not a sequence, SEQNUM is 1.
# SEQTOT, int, The total number of exposures taken in the current sequence, as given by ICS. If not a sequence, SEQTOT is 1.
# PROGRAM, string, The program as given by ICS.
# MJD-OBS, float, The MJD-OBS as given by ICS. Modified Julian Date of the observation.
# BADCAMWORD, string, camword defining the bad cameras that should not be processed.
# BADAMPS, string, comma separated list of "{camera}{petal}{amp}", i.e. "[brz][0-9][ABCD]". Example: 'b7D,z8A'
#                  in the csv this is saved as a semicolon separated list
# LASTSTEP, string, the last step the pipeline should run through for the given exposure. Inclusive of last step.
# EXPFLAG, np.ndarray, set of flags that describe that describe the exposure.
# HEADERERR, np.ndarray, In the csv given as a "|" separated list of key=value pairs describing columns in the table that should
#                        be corrected. The workflow transforms these into an array of strings.
#                        NOTE: This will be used to change the given key/value pairs in the production table.
# COMMENTS, np.ndarray, In the csv given as either a "|" separated list of comments or one long comment. When loaded it
#                       is a numpy array of the strings.These are not used by the workflow but useful for humans
#                       to put notes for other humans.
##################################################

def exposure_table_column_defs():
    """
    Contains the column names, data types, and default row values for a DESI Exposure table. It returns
    the names, datatypes, and defaults as a list of 3-tuples.

    Args:
        None.

    Returns:
        columns, list. List of tuples (name, type, default).
    """
    columns = [
                ('EXPID', int, -99),
                ('OBSTYPE', 'S8', 'unknown'),
                ('TILEID', int, -99),
                ('LASTSTEP', 'S30', 'all'),
                ('CAMWORD', 'S30', 'a0123456789'),
                ('BADCAMWORD', 'S30', ''),
                ('BADAMPS', 'S30', ''),
                ('EXPTIME', float, 0.0),
                ('EFFTIME_ETC', float, -99.),
                ('SURVEY', 'S10', 'unknown'),
                ('FA_SURV', 'S10', 'unknown'),
                ('FAPRGRM', 'S20', 'unknown'),
                ('GOALTIME', float, -99.),
                ('GOALTYPE', 'S10', 'unknown'),
                ('EBVFAC', float, 1.0),
                ('AIRMASS', float, 1.0),
                ('SPEED', float, -99.0),
                ('TARGTRA', float, 89.99),
                ('TARGTDEC', float, -89.99),
                ('SEQNUM', int, 1),
                ('SEQTOT', int, 1),
                ('PROGRAM', 'S60', 'unknown'),
                ('PURPOSE', 'S30', 'unknown'),
                ('MJD-OBS', float, 50000.0),
                ('NIGHT', int, 20000101),
                ('HEADERERR', np.ndarray, np.array([], dtype=str)),
                ('EXPFLAG', np.ndarray, np.array([], dtype=str)),
                ('COMMENTS', np.ndarray, np.array([], dtype=str))
              ]
    return columns

def default_obstypes_for_exptable():
    """
    Defines the exposure types to be recognized by the workflow and saved in the exposure table by default.

{'fraction_non_alphanumeric': 0.13461335583922354, 'fraction_numerical': 0.017301403101592994, 'mean_word_length': 2.647556752597153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4801597', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2671, 'n_words': 1082}