<commit_before>#
# A class to merge quicklook qa outputs.
#  
from __future__ import absolute_import, division, print_function
from desiutil.io import yamlify
import yaml
import json
import numpy as np

###################################
def delKey(d, k, val=None, remove=True):
    
    try:
        for key, value in  d.iteritems():  # if it's a dictionary
           if key==k:
              if remove: del d[key]
              val = value
           val = delKey(value, k, val=val, remove=remove)
    except: 
        try: 
          for i in range(len(d)):  # if it's a list
             val = delKey(d[i], k, val=val, remove=remove)
        except:
            return val
        
    return val
###################################
def reOrderDict(mergeDict):
    
  for Night in mergeDict["NIGHTS"]:
      for Exposure in Night["EXPOSURES"]:
          for Camera in Exposure["CAMERAS"]:

             ra  = delKey(Camera, "RA")
             dec = delKey(Camera, "DEC")
             sky_fiberid = delKey(Camera, "SKY_FIBERID")
             skyfiberid = delKey(Camera, "SKYFIBERID")
             
             if sky_fiberid is None:
                 sky_fiberid = skyfiberid
             
             elg_fiberid = delKey(Camera, "ELG_FIBERID")
             lrg_fiberid = delKey(Camera, "LRG_FIBERID") 
             qso_fiberid = delKey(Camera, "QSO_FIBERID") 
             star_fiberid = delKey(Camera, "STAR_FIBERID", remove=False)
             
             std_fiberid = delKey(Camera, "STD_FIBERID", remove=False)
             
             if star_fiberid is None:
                 star_fiberid = std_fiberid
             
             b_peaks = delKey(Camera, "B_PEAKS") 
             r_peaks = delKey(Camera, "R_PEAKS")
             z_peaks = delKey(Camera, "Z_PEAKS")
            
             try: ra = [float("%.5f" % m) for m in ra]
             except: ra=None
             
             try: dec = [float("%.5f" % m) for m in dec]
             except: dec=None

             #- Add imaging mag by hand to this to facilitate QLF implementation. Should come from calculation/headers... somewhere.
             imaging_mag=[22.]*len(ra) #- same length as objects(ra)
             
             Camera["GENERAL_INFO"]={"RA":ra, "DEC":dec, "SKY_FIBERID":sky_fiberid, "ELG_FIBERID":elg_fiberid ,"LRG_FIBERID":lrg_fiberid, "QSO_FIBERID":qso_fiberid ,"STAR_FIBERID":star_fiberid ,"B_PEAKS":b_peaks ,"R_PEAKS":r_peaks ,"Z_PEAKS":z_peaks, "IMAGING_MAG": imaging_mag}   
###################################

class QL_QAMerger:
    def __init__(self,night,expid,flavor,camera,program):
        self.__night=night
        self.__expid=expid
        self.__flavor=flavor
        self.__camera=camera
        self.__program=program
        self.__stepsArr=[]
        self.__schema={'NIGHTS':[{'NIGHT':night,'EXPOSURES':[{'EXPID':expid,'FLAVOR':flavor,'PROGRAM':program, 'CAMERAS':[{'CAMERA':camera, 'PIPELINE_STEPS':self.__stepsArr}]}]}]}
        
        
    class QL_Step:
        def __init__(self,paName,paramsDict,metricsDict):
            self.__paName=paName
            self.__pDict=paramsDict
            self.__mDict=metricsDict<commit_msg>avoid None for list in the imaging mag<commit_after>#
# A class to merge quicklook qa outputs.
#  
from __future__ import absolute_import, division, print_function
from desiutil.io import yamlify
import yaml
import json
import numpy as np

###################################
def delKey(d, k, val=None, remove=True):
    
    try:
        for key, value in  d.iteritems():  # if it's a dictionary
           if key==k:
              if remove: del d[key]
              val = value
           val = delKey(value, k, val=val, remove=remove)
    except: 
        try: 
          for i in range(len(d)):  # if it's a list
             val = delKey(d[i], k, val=val, remove=remove)
        except:
            return val
        
    return val
###################################
def reOrderDict(mergeDict):
    
  for Night in mergeDict["NIGHTS"]:
      for Exposure in Night["EXPOSURES"]:
          for Camera in Exposure["CAMERAS"]:

             ra  = delKey(Camera, "RA")
             dec = delKey(Camera, "DEC")
             sky_fiberid = delKey(Camera, "SKY_FIBERID")
             skyfiberid = delKey(Camera, "SKYFIBERID")
             
             if sky_fiberid is None:
                 sky_fiberid = skyfiberid
             
             elg_fiberid = delKey(Camera, "ELG_FIBERID")
             lrg_fiberid = delKey(Camera, "LRG_FIBERID") 
             qso_fiberid = delKey(Camera, "QSO_FIBERID") 
             star_fiberid = delKey(Camera, "STAR_FIBERID", remove=False)
             
             std_fiberid = delKey(Camera, "STD_FIBERID", remove=False)
             
             if star_fiberid is None:
                 star_fiberid = std_fiberid
             
             b_peaks = delKey(Camera, "B_PEAKS") 
             r_peaks = delKey(Camera, "R_PEAKS")
             z_peaks = delKey(Camera, "Z_PEAKS")
            
             try: ra = [float("%.5f" % m) for m in ra]
             except: ra=None
             
             try: dec = [float("%.5f" % m) for m in dec]
             except: dec=None

             #- Add imaging mag by hand to this to facilitate QLF implementation. Should come from calculation/headers... somewhere.
             imaging_mag=[22.]*500 #- should be nfibers. Get from somewhere or allow None for ra (look except above) in test_ql.py mergeQA test. 
             
             Camera["GENERAL_INFO"]={"RA":ra, "DEC":dec, "SKY_FIBERID":sky_fiberid, "ELG_FIBERID":elg_fiberid ,"LRG_FIBERID":lrg_fiberid, "QSO_FIBERID":qso_fiberid ,"STAR_FIBERID":star_fiberid ,"B_PEAKS":b_peaks ,"R_PEAKS":r_peaks ,"Z_PEAKS":z_peaks, "IMAGING_MAG": imaging_mag}   
###################################

class QL_QAMerger:
    def __init__(self,night,expid,flavor,camera,program):
        self.__night=night
        self.__expid=expid
        self.__flavor=flavor
        self.__camera=camera
        self.__program=program
        self.__stepsArr=[]
        self.__schema={'NIGHTS':[{'NIGHT':night,'EXPOSURES':[{'EXPID':expid,'FLAVOR':flavor,'PROGRAM':program, 'CAMERAS':[{'CAMERA':camera, 'PIPELINE_STEPS':self.__stepsArr}]}]}]}
        
        
    class QL_Step:
        def __init__(self,paName,paramsDict,metricsDict):
            self.__paName=paName
            self.__pDict=paramsDict
            self.__mDict=metricsDict
{'fraction_non_alphanumeric': 0.13851667186039265, 'fraction_numerical': 0.0017139295730757246, 'mean_word_length': 2.2159318637274548, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2990986', 'n_tokens_mistral': 2114, 'n_tokens_neox': 2064, 'n_words': 442}