<commit_before>
''' General Utilities for MaNGA SAS'''

import json, os, glob
from flask import session as current_session
from ast import literal_eval
from manga_utils import generalUtils as gu
from astropy.table import Table
from model.database import db
from jinja_filters import getMPL
import sdss.internal.database.utah.mangadb.DataModelClasses as datadb

def getMaskBitLabel(bits):
    ''' Return a list of flag names '''
    
    session = db.Session()
    flagnames=[]
    for bit in bits:
        if bit: 
            try: 
                masklabels = session.query(datadb.MaskLabels).filter_by(maskbit=long(bit)).one()
                labels = json.loads(masklabels.labels)
                labels = str(','.join(labels))
            except:
                labels = None
        else: labels = None
        flagnames.append(labels)

    return flagnames

def makeQualNames(bits, stage='2d'):
    ''' Return list containing the quality flag names '''
     
    name = 'MANGA_DRP2QUAL' if stage=='2d' else 'MANGA_DRP3QUAL'
    #flagnames = [gu.getSDSSFlagName(bit,name=name) for bit in bits]
    flagnames = [getFlags(bit,name) for bit in bits]
    return flagnames
    
def getTargNames(bits, type=1):
    ''' Return list containing MaNGA Target Flags '''
    
    name = 'MANGA_TARGET1' if type==1 else 'MANGA_TARGET2' if type==2 else 'MANGA_TARGET3'
    #flagnames = [gu.getSDSSFlagName(bit,name=name) for bit in bits]
    flagnames = [getFlags(bit,name) for bit in bits]
    return flagnames

def getFlags(bits,name):
    ''' Get the labels for a given bitmask, from database '''

    session = db.Session()	    
    
    # if bits not a digit, return None
    if not str(bits).isdigit(): return 'NULL'
    else: bits=int(bits)
    
    # Convert the integer value to list of bits
    bitlist=[int(i) for i in '{0:08b}'.format(bits)]
    bitlist.reverse()
    indices = [i for i,bit in enumerate(bitlist) if bit]	    
    
    labels=[]
    for i in indices:
        maskbit = session.query(datadb.MaskBit).filter_by(flag=name,bit=i).one()
        labels.append(str(maskbit.label))    
    
    if not labels: labels=None
    
    return labels
    
    
def getColumnTypes(columns):
    ''' Return the data types for table columns '''
    
    typedict = {'nexp':int, 'exptime':float, 'bluesn2':float, 'redsn2':float,
        'airmsmin':float, 'airmsmed':float, 'airmsmax':float, 'mjdmin':long, 
        'mjdmed':long, 'mjdmax':long, 'objra':float, 'objdec':float,
        'ifudesign':str, 'harname':str, 'cenra':float, 'cendec':float,
        'frlplug':int, 'mangaid':str, 'ufwhm':float,'gfwhm':float,
        'rfwhm':float,'ifwhm':float,'zfwhm':float, 'ebvgal':float, 'datered':long,'cartid':str,
        'versdrp3':str,'verscore':str, 'versutil':str, 'drp3qual':long,
        'objglon':float, 'objglat':float, 'designid':long,'catidnum':long,
        'seemin':float, 'seemed':float, 'seemax':float,'transmin':float, 'transmed':float, 
        'transmax':float, 'mngtarg1':long, 'mngtarg2':long, 'plttarg':str,'mjdred':str,
        'mngtarg3':long, 'mangaid':str, 'plate':long, 'platetyp':str, 'srvymode':str, 'versdrp2':str,
        'ifura':float,'ifudec':float,'ifuglon':float,'ifuglat':float}
    
    types = [typedict[col] if col in typedict.keys() else str for col in columns]
        
    return types, typedict    
    
def processTableData(tableobj):
    ''' Process a JSON object table and convert to an Astropy Table '''
    
    table = (json.loads(tableobj))
    nrows = len(table)

    keys = [k for k in table[0].keys() if '_class' not in k]
    types,typedict = getColumnTypes(keys)
   
    for row in table:
        for key,val in row.items():
            # pull the plate id from the link
            index = val.find('">')
            if index != -1: row[key] = val[index+2:index+6]
            
            # convert QUAL keys back to bits
            if 'qual' in key: row[key] = gu.getSDSSFlagBit(literal_eval(row[key]),name='MANGA_DRP3QUAL') if row[key] != 'NULL' else None
            
            # reset NULL values to the right type
            if val == None or val=='None' or val=='NULL':
                row[key] = gu.getNullVal(typedict[key])
        
        # Delete the _class key
        tmp = row.pop('_class',None) 
    
    newtable = Table(table,dtype=types,names=keys)    
    
    return newtable
    
    
def getDRPVersion():
    ''' Get DRP version to use during MaNGA SAS '''

	# DRP versions
    session = db.Session()    
    vers = session.query(datadb.PipelineVersion).filter(datadb.PipelineVersion.version.like('%v%')).order_by(datadb.PipelineVersion.version.desc()).all()
    versions = [v.version for v in vers]
    
    return versions    

def getDAPVersion():
    ''' Get DAP version to use during MaNGA SAS '''
    
    # DAP versions
    session = db.Session()    
    vers = session.query(datadb.PipelineVersion).join(datadb.PipelineInfo,datadb.PipelineName).\
    filter(datadb.PipelineName.label=='DAP',datadb.PipelineVersion.version.like('%v%')).\
    order_by(datadb.PipelineVersion.version.desc()).all()
    versions = [v.version for v in vers]
    
    return versions+['NA']

def setMPLVersion(mplver):
    ''' set the versions based on MPL '''
    
    mpl = getMPL(mplver)
    drpver,dapver = mpl.split(':')[1].strip().split(', ')
    current_session['currentver'] = drpver
    current_session['currentdapver'] = dapver if dapver != 'NA' else None
    
def setGlobalVersion():
    ''' set the global version '''

    # set MPL version
    try: mplver = current_session['currentmpl']
    except: mplver = None
    if not mplver: current_session['currentmpl']='MPL-3'
    
    # set version mode
    try: vermode = current_session['vermode']
    except: vermode = None 
    if not vermode: current_session['vermode']='MPL'
    
    # initialize
    if 'MPL' in current_session['vermode']:
        setMPLVersion(current_session['currentmpl'])
    
    # set global DRP version
    try: versions = current_session['versions'] 
    except: versions = getDRPVersion()
    current_session['versions'] = versions
    try: drpver = current_session['currentver']
    except: drpver = None
    if not drpver: 
        realvers = [ver for ver in versions if os.path.isdir(os.path.join(os.getenv('MANGA_SPECTRO_REDUX'),ver))]
        current_session['currentver'] = realvers[0]
        
    # set global DAP version
    try: dapversions = current_session['dapversions'] 
    except: dapversions = getDAPVersion()
    current_session['dapversions'] = dapversions
    try: ver = current_session['currentdapver']
    except: ver = None<commit_msg>testing marvin for MPL-4 readiness<commit_after>
''' General Utilities for MaNGA SAS'''

import json, os, glob
from flask import session as current_session
from ast import literal_eval
from manga_utils import generalUtils as gu
from astropy.table import Table
from model.database import db
from jinja_filters import getMPL
import sdss.internal.database.utah.mangadb.DataModelClasses as datadb

def getMaskBitLabel(bits):
    ''' Return a list of flag names '''
    
    session = db.Session()
    flagnames=[]
    for bit in bits:
        if bit: 
            try: 
                masklabels = session.query(datadb.MaskLabels).filter_by(maskbit=long(bit)).one()
                labels = json.loads(masklabels.labels)
                labels = str(','.join(labels))
            except:
                labels = None
        else: labels = None
        flagnames.append(labels)

    return flagnames

def makeQualNames(bits, stage='2d'):
    ''' Return list containing the quality flag names '''
     
    name = 'MANGA_DRP2QUAL' if stage=='2d' else 'MANGA_DRP3QUAL'
    #flagnames = [gu.getSDSSFlagName(bit,name=name) for bit in bits]
    flagnames = [getFlags(bit,name) for bit in bits]
    return flagnames
    
def getTargNames(bits, type=1):
    ''' Return list containing MaNGA Target Flags '''
    
    name = 'MANGA_TARGET1' if type==1 else 'MANGA_TARGET2' if type==2 else 'MANGA_TARGET3'
    #flagnames = [gu.getSDSSFlagName(bit,name=name) for bit in bits]
    flagnames = [getFlags(bit,name) for bit in bits]
    return flagnames

def getFlags(bits,name):
    ''' Get the labels for a given bitmask, from database '''

    session = db.Session()	    
    
    # if bits not a digit, return None
    if not str(bits).isdigit(): return 'NULL'
    else: bits=int(bits)
    
    # Convert the integer value to list of bits
    bitlist=[int(i) for i in '{0:08b}'.format(bits)]
    bitlist.reverse()
    indices = [i for i,bit in enumerate(bitlist) if bit]	    
    
    labels=[]
    for i in indices:
        maskbit = session.query(datadb.MaskBit).filter_by(flag=name,bit=i).one()
        labels.append(str(maskbit.label))    
    
    if not labels: labels=None
    
    return labels
    
    
def getColumnTypes(columns):
    ''' Return the data types for table columns '''
    
    typedict = {'nexp':int, 'exptime':float, 'bluesn2':float, 'redsn2':float,
        'airmsmin':float, 'airmsmed':float, 'airmsmax':float, 'mjdmin':long, 
        'mjdmed':long, 'mjdmax':long, 'objra':float, 'objdec':float,
        'ifudesign':str, 'harname':str, 'cenra':float, 'cendec':float,
        'frlplug':int, 'mangaid':str, 'ufwhm':float,'gfwhm':float,
        'rfwhm':float,'ifwhm':float,'zfwhm':float, 'ebvgal':float, 'datered':long,'cartid':str,
        'versdrp3':str,'verscore':str, 'versutil':str, 'drp3qual':long,
        'objglon':float, 'objglat':float, 'designid':long,'catidnum':long,
        'seemin':float, 'seemed':float, 'seemax':float,'transmin':float, 'transmed':float, 
        'transmax':float, 'mngtarg1':long, 'mngtarg2':long, 'plttarg':str,'mjdred':str,
        'mngtarg3':long, 'mangaid':str, 'plate':long, 'platetyp':str, 'srvymode':str, 'versdrp2':str,
        'ifura':float,'ifudec':float,'ifuglon':float,'ifuglat':float}
    
    types = [typedict[col] if col in typedict.keys() else str for col in columns]
        
    return types, typedict    
    
def processTableData(tableobj):
    ''' Process a JSON object table and convert to an Astropy Table '''
    
    table = (json.loads(tableobj))
    nrows = len(table)

    keys = [k for k in table[0].keys() if '_class' not in k]
    types,typedict = getColumnTypes(keys)
   
    for row in table:
        for key,val in row.items():
            # pull the plate id from the link
            index = val.find('">')
            if index != -1: row[key] = val[index+2:index+6]
            
            # convert QUAL keys back to bits
            if 'qual' in key: row[key] = gu.getSDSSFlagBit(literal_eval(row[key]),name='MANGA_DRP3QUAL') if row[key] != 'NULL' else None
            
            # reset NULL values to the right type
            if val == None or val=='None' or val=='NULL':
                row[key] = gu.getNullVal(typedict[key])
        
        # Delete the _class key
        tmp = row.pop('_class',None) 
    
    newtable = Table(table,dtype=types,names=keys)    
    
    return newtable
    
    
def getDRPVersion():
    ''' Get DRP version to use during MaNGA SAS '''

	# DRP versions
    session = db.Session()    
    vers = session.query(datadb.PipelineVersion).filter(datadb.PipelineVersion.version.like('%v%')).order_by(datadb.PipelineVersion.version.desc()).all()
    versions = [v.version for v in vers]
    
    return versions    

def getDAPVersion():
    ''' Get DAP version to use during MaNGA SAS '''
    
    # DAP versions
    session = db.Session()    
    vers = session.query(datadb.PipelineVersion).join(datadb.PipelineInfo,datadb.PipelineName).\
    filter(datadb.PipelineName.label=='DAP',datadb.PipelineVersion.version.like('%v%')).\
    order_by(datadb.PipelineVersion.version.desc()).all()
    versions = [v.version for v in vers]
    
    return versions+['NA']

def setMPLVersion(mplver):
    ''' set the versions based on MPL '''
    
    mpl = getMPL(mplver)
    drpver,dapver = mpl.split(':')[1].strip().split(', ')
    current_session['currentver'] = drpver
    current_session['currentdapver'] = dapver if dapver != 'NA' else None
    
def setGlobalVersion():
    ''' set the global version '''

    # set MPL version
    try: mplver = current_session['currentmpl']
    except: mplver = None
    if not mplver: current_session['currentmpl']='MPL-4'
    
    # set version mode
    try: vermode = current_session['vermode']
    except: vermode = None 
    if not vermode: current_session['vermode']='MPL'
    
    # initialize
    if 'MPL' in current_session['vermode']:
        setMPLVersion(current_session['currentmpl'])
    
    # set global DRP version
    try: versions = current_session['versions'] 
    except: versions = getDRPVersion()
    current_session['versions'] = versions
    try: drpver = current_session['currentver']
    except: drpver = None
    if not drpver: 
        realvers = [ver for ver in versions if os.path.isdir(os.path.join(os.getenv('MANGA_SPECTRO_REDUX'),ver))]
        current_session['currentver'] = realvers[0]
        
    # set global DAP version
    try: dapversions = current_session['dapversions'] 
    except: dapversions = getDAPVersion()
    current_session['dapversions'] = dapversions
    try: ver = current_session['currentdapver']
    except: ver = None
{'fraction_non_alphanumeric': 0.13080232470375122, 'fraction_numerical': 0.004453166276700129, 'mean_word_length': 3.051987767584098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4419889', 'n_tokens_mistral': 4394, 'n_tokens_neox': 4310, 'n_words': 1047}