<commit_before>import sys
import os
sys.path.insert(0,os.path.dirname(sys.executable)+'/../lib/python2.7/site-packages/') #ignore other PYTHONPATH

import logging
import docopt
import schema
import brilws
import prettytable
import pandas as pd
import numpy as np
import ast
from brilws import api,params,display,formatter,lumiParameters,corrector
from brilws.cli import clicommonargs
import re,time, csv
from datetime import datetime
from sqlalchemy import *
import math
from itertools import izip_longest
from dateutil import tz
import pytz
log = logging.getLogger('brilws')
logformatter = logging.Formatter('%(levelname)s %(name)s %(message)s')
log.setLevel(logging.ERROR)
ch = logging.StreamHandler()
ch.setFormatter(logformatter)
log.addHandler(ch)

lumip = lumiParameters.ParametersObject()
lslengthsec= lumip.lslengthsec()
utctmzone = tz.gettz('UTC')
cerntmzone = tz.gettz('CEST')

class Unbuffered(object):
    def __init__(self, stream):
        self.stream = stream
    def write(self, data):
        self.stream.write(data)
        self.stream.flush()
    def __getattr__(self, attr):
        return getattr(self.stream.attr)
sys.stdout = Unbuffered(sys.stdout)

np.seterr(divide='ignore', invalid='ignore')

def xing_indexfilter(arr,constfactor=1.,xingMin=0.,xingTr=0.,xingId=[]):
    vidx = None    
    if xingTr:
        vidx = np.argwhere( np.logical_and( (arr*constfactor)>xingMin, arr>xingTr*np.max(arr) ) ).ravel()
    else:
        vidx = np.argwhere( (arr*constfactor)>xingMin ).ravel()
    if xingId :
        posidx = np.array(xingId)-1 #array position index is bunch index - 1
        return np.intersect1d(np.array(posidx),vidx)    
    return vidx

def totalprescaleNEW(hltprescval,l1seedlogic,l1prescvals):
    '''
    1) if ONE single seed, take it
    2) if OR:  (A.1 OR A.2 OR A.3 ...) ignore the triggers with prescale 0, and take the minimum of the remaining prescale values:
        if max(A.i) == 0
                A = 0
        else
                A = min(A.i | A.i > 0)
    3) if AND: A AND B AND C ... take the product of the prescale values A * B * C ...
    4) (optional)
       if (A.1 OR A.2 OR A.3 ...) AND (B.1 OR B.2 OR ...) AND ...
       - for each group, take the lowest non-zero prescale, or 0 if they are all zero
         if max(A.i) = 0
                A =
         else
                A = min(A.i | A.i > 0)
       - take the product of the prescales thus obtained:
         P = A * B * ...
       users still take their own risk and should use their own judgement.
    5)  if NOT, or a combination of AND/OR and NOT, reject
    '''
    totpresc = 0
    if not hltprescval or not l1prescvals or not l1seedlogic:
        return totpresc
    if np.all( np.array(l1prescvals)==1 ):
        totpresc = hltprescval        
    elif l1seedlogic=='ONE':
        totpresc = hltprescval*l1prescvals[0]
    elif l1seedlogic=='OR':
        if np.max( np.array(l1prescvals) ) == 0:
            return totpresc
        else:
            l = np.array(l1prescvals)
            l1p = np.min( l[l>0] )
        totpresc = hltprescval*l1p
    elif l1seedlogic=='AND':
        if np.max( np.array(l1prescvals) ) == 0:
            return totpresc
        l1p = np.prod( np.array(l1prescvals) )
        totpresc = hltprescval*l1p    
    return totpresc

    
def totalprescale(hltprescval,l1seedlogic,l1prescvals):
    totpresc = 0
    if not hltprescval or not l1prescvals or not l1seedlogic:
        return totpresc
    if np.all( np.array(l1prescvals)==1 ):
        totpresc = hltprescval
    elif l1seedlogic=='ONE':
        totpresc = hltprescval*l1prescvals[0]
    elif l1seedlogic=='OR':
        
        totpresc = hltprescval*np.min(l1prescvals)
    elif l1seedlogic=='AND':
        totpresc = hltprescval*np.max(l1prescvals)    
    return totpresc

def lumi_per_normtag(shards,lumiquerytype,dbengine,dbschema,runtot,datasource=None,normtag=None,withBX=False,byls=None,fh=None,csvwriter=None,ptable=None,scalefactor=1,totz=utctmzone,fillmin=None,fillmax=None,runmin=None,runmax=None,amodetagid=None,egev=None,beamstatusid=None,tssecmin=None,tssecmax=None,runlsSeries=None,hltl1map={},ignorel1mask=False,xingMin=0.,xingTr=0.,xingId=[],checkjson=False,datatagnameid=None):

    validitychecker = None
    lastvalidity = None
    if normtag and normtag is not 'withoutcorrection':
        normdata = api.iov_gettagdata(dbengine, normtag,schemaname=dbschema)
        if not normdata: raise ValueError('normtag %s does not exist'%normtag)
        validitychecker = ValidityChecker(normdata)
        
    for shard in shards:              
        lumiiter = None
        if lumiquerytype == 'bestresultonline':
            tablename = 'online_result_'+str(shard)
            shardexists = api.table_exists(dbengine,tablename,schemaname=dbschema)
            if not shardexists: continue
            rfields = ['fillnum','runnum','lsnum','timestampsec','cmson','beamstatusid','targetegev','delivered','recorded','avgpu','datasource','numbxbeamactive']
            if withBX: rfields = rfields+['bxdeliveredblob']                    
            lumiiter = api.online_resultIter(dbengine,tablename,schemaname=dbschema,fields=rfields,fillmin=fillmin,fillmax=fillmax,runmin=runmin,runmax=runmax,amodetagid=amodetagid,targetegev=egev,beamstatusid=beamstatusid,tssecmin=tssecmin,tssecmax=tssecmax,runlsselect=runlsSeries,sorted=True)
            
        elif lumiquerytype in ['detresultonline','detraw']:
            datatype = 'raw'
            if lumiquerytype=='detresultonline':
                datatype = 'result'
                
            tablename = datasource.lower()+'_'+datatype+'_'+str(shard)
            shardexists = api.table_exists(dbengine,tablename,schemaname=dbschema)
            if not shardexists: continue
            rfields = ['avglumi']
            idfields = ['fillnum','runnum','lsnum','timestampsec','beamstatusid','cmson','deadtimefrac','targetegev','numbxbeamactive']
            if withBX: rfields = rfields+['bxlumiblob']
            lumiiter = api.dataIter(dbengine,datasource,datatype,shard,datafields=rfields,idfields=idfields,schemaname=dbschema,fillmin=fillmin,fillmax=fillmax,runmin=runmin,runmax=runmax,amodetagid=amodetagid,targetegev=egev,beamstatusid=beamstatusid,tssecmin=tssecmin,tssecmax=tssecmax,runlsselect=runlsSeries,sorted=True,datatagnameid=datatagnameid)                    
                  
        if lumiiter is None:
            continue

        g_run_old = 0
        g_ls_trglastscaled_old = 0
        prescale_map = {} # for global scope
        g_hltconfigid = 0
        for row in lumiiter:
            fillnum = row['fillnum']
            runnum = row['runnum']
            lsnum = row['lsnum']            
            cmslsnum = lsnum
            timestampsec = row['timestampsec']
            cmson = row['cmson']
            if not cmson:
                cmslsnum = 0
            if hltl1map:
                if cmslsnum==0: continue  #cms is not running, skip.                
                if g_run_old!=runnum:     #on new run boundary, get hltconfigid
                    hltrunconfig = api.get_hltrunconfig(dbengine,runnum=runnum,schemaname=dbschema) #[['runnum','hltconfigid','hltkey']]
                    if not hltrunconfig:
                        continue
                    g_hltconfigid = [h[1] for h in hltrunconfig][0]
                    if not g_hltconfigid:
                        continue      
                    presc = api.get_prescidx_change(dbengine,runnum,schemaname=dbschema)#{runnum:[[lslastscaler,prescidx]] }
                    if presc.has_key(runnum):
                        presc = presc[runnum]
                this_prescidx = None
                if not presc:                    
                    ls_trglastscaled = 1
                else:
                    b = [ i[0] for i in presc if i[0]<=cmslsnum ] 
                    if not b:
                        ls_trglastscaled = 1
                        this_prescidx = None                        
                    else:
                        ls_trglastscaled = np.max( b )
                        this_prescidx = [t[1] for t in presc if t[0]==ls_trglastscaled][0]
                if (g_ls_trglastscaled_old != ls_trglastscaled) or (g_run_old != runnum): #on prescale change lumi section or new run
                    prescale_map = {} #clear prescaleindex change map on new prescale index value
                    if not hltl1map.has_key(g_hltconfigid):
                        continue
                    this_hltl1map = hltl1map[g_hltconfigid]#[[hltpathid,hltpathname,l1seedtype,l1seedbits]]
                    for grouped in this_hltl1map:
                        this_hltpathid = grouped[0]
                        hltpathname = grouped[1]
                        l1seedlogic = grouped[2]
                        l1candidates = grouped[3]
                        if this_prescidx is None:
                            hltprescval = 1
                            totpresc = 1
                        else:
                            hltprescval = api.get_hltprescale(dbengine,runnum,ls_trglastscaled,g_hltconfigid,this_prescidx,this_hltpathid,schemaname=dbschema)
                            rl = api.get_l1prescale(dbengine,runnum,ls_trglastscaled,l1candidates=l1candidates,prescidxs=this_prescidx,ignorel1mask=ignorel1mask ,schemaname=dbschema)
                            if not rl: continue
                            l1keys = [ k for k in rl.keys() if k[0]==this_prescidx ]
                            l1vals = [ rl[k] for k in l1keys ]                       
                            l1prescvals = [ v[0] for v in l1vals ]                            
                            totpresc = totalprescaleNEW(hltprescval,l1seedlogic,l1prescvals)
                            if not totpresc:
                                continue
                        prescale_map[hltpathname] = totpresc   
                    g_ls_trglastscaled_old = ls_trglastscaled
                g_run_old = runnum
            beamstatusid = row['beamstatusid']
            beamstatus = params._idtobeamstatus[beamstatusid]
            if beamstatus not in ['FLAT TOP','STABLE BEAMS','SQUEEZE','ADJUST']: continue
            tegev = row['targetegev']
            dtime = str(timestampsec)
            if totz is not None:
                d = datetime.fromtimestamp(int(timestampsec),tz=pytz.utc)
                dtime = d.astimezone(totz).strftime(params._datetimefm)
            delivered = recorded = avgpu = livefrac = 0.
            if checkjson:
                g_returnedls.append((runnum,lsnum))
            if lumiquerytype == 'bestresultonline': ##bestlumi
                if row.has_key('delivered') and row['delivered']:                    
                    delivered = np.true_divide(row['delivered']*lslengthsec,scalefactor)
                if delivered>0 and row.has_key('recorded') and row['recorded']:
                    recorded = np.true_divide(row['recorded']*lslengthsec,scalefactor)
                if delivered>0 and row.has_key('avgpu') and row['avgpu']:
                    avgpu = row['avgpu']
                ds = 'UNKNOWN' 
                if row.has_key('datasource') and row['datasource']:
                    ds = row['datasource']
                livefrac = np.true_divide(recorded,delivered)
                if withBX:    #--xing
                    bxlumi = None
                    bxlumistr = '[]'
                    if row.has_key('bxdeliveredblob'):
                        bxdeliveredarray = np.array(api.unpackBlobtoArray(row['bxdeliveredblob'],'f'))                        
                        totfactor = np.true_divide(lslengthsec,scalefactor)
                        bxidx = xing_indexfilter(bxdeliveredarray,constfactor=totfactor,xingMin=xingMin,xingTr=xingTr,xingId=xingId)
                        if bxidx is not None and bxidx.size>0:
                            bxdelivered = bxdeliveredarray[bxidx]*totfactor
                            bxlumi = np.transpose( np.array([bxidx+1,bxdelivered,bxdelivered*livefrac]) )
                        del bxdeliveredarray
                        del bxidx
                        if hltl1map: #--hltpath bx display
                            for pth in prescale_map.keys():                                
                                thispresc = prescale_map[pth]
                                if bxlumi is not None:
                                    a = map(formatter.bxlumi,bxlumi/thispresc)  
                                    bxlumistr = '['+' '.join(a)+']'                                
                                display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),ds,'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                                
                        else:        #normal bx display                   
                            if bxlumi is not None:
                                a = map(formatter.bxlumi,bxlumi)
                                bxlumistr = '['+' '.join(a)+']'
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),ds,'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    del bxlumi
                    
                elif byls:      #--byls
                    if hltl1map:#--hltpath display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum), dtime, pth, '%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),'%.1f'%(avgpu),ds] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:       #normal display
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),ds] , fh=fh, csvwriter=csvwriter, ptable=ptable)                        
            else:               ###lumisource
                if row.has_key('deadtimefrac') :
                    if row['deadtimefrac'] is not None:
                        livefrac = 1.-row['deadtimefrac']
                    else:
                        if cmslsnum!=0:
                            g_nulldeadtime.setdefault(runnum,[]).append(lsnum)
                        livefrac = 0.
                uncorrectedavglumi = row['avglumi'] #uncorrected, hz/ub
                avglumi = uncorrectedavglumi
                ncollidingbx = row['numbxbeamactive']
                uncorrectedbxlumi = None
                if row.has_key('bxlumiblob'):
                    uncorrectedbxlumi = np.array(api.unpackBlobtoArray(row['bxlumiblob'],'f'))
                bxlumi = uncorrectedbxlumi
                if validitychecker is not None:
                    if not lastvalidity or not validitychecker.isvalid(runnum,lastvalidity):
                        lastvalidity = validitychecker.getvalidity(runnum)
                    [normfunc,normparam] = validitychecker.getvaliddata(lastvalidity[0])
                    if withBX:
                        (avglumi,bxlumi) = corrector.applyCorrectionStr( normfunc,normparam, corrector.FunctionRoot(uncorrectedbxlumi,avglumi,ncollidingbx) )
                    else:
                        avglumi = corrector.applyCorrectionStr( normfunc,normparam, corrector.FunctionRoot(avglumi,avglumi,ncollidingbx) )[0]

                totfactor = np.true_divide(lslengthsec,scalefactor)
                delivered = avglumi*totfactor
                recorded = delivered*livefrac
                avgpu = lumip.avgpu( avglumi,ncollidingbx,g_minbias)                
                if withBX:      #--xing
                    bxlumistr = '[]'
                    if bxlumi is not None:                                                         
                        bxidx = xing_indexfilter(bxlumi,constfactor=totfactor,xingMin=xingMin,xingTr=xingTr,xingId=xingId)
                        if bxidx is not None and bxidx.size>0:                                      
                            bxdelivered =  bxlumi[bxidx]*totfactor
                            bxlumi = np.transpose( np.array([bxidx+1,bxdelivered,bxdelivered*livefrac]) )               
                        del uncorrectedbxlumi
                        del bxidx
                    if hltl1map:#--hltpath xing display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]                            
                            if bxlumi is not None:
                                a = map(formatter.bxlumi,bxlumi/thispresc)  
                                bxlumistr = '['+' '.join(a)+']'                                
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),datasource.upper(),'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:       #normal xing display
                        if bxlumi is not None:
                            a = map(formatter.bxlumi,bxlumi)
                            bxlumistr = '['+' '.join(a)+']'
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),datasource.upper(),'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                        
                    del bxlumi
                elif byls:       #--byls
                    if hltl1map: #--hltpath display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]                            
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),'%.1f'%(avgpu),datasource.upper()] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:        #normal display
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),datasource.upper()] , fh=fh, csvwriter=csvwriter, ptable=ptable)
            if not hltl1map:     #normal statistic collect
                if not runtot.has_key( ('',runnum) ):
                    runtot[('',runnum)] = {'fill':fillnum,'dtime':dtime,'nls':0,'ncms':0,'delivered':0,'recorded':0} <commit_msg>fix bxlumi display when no bunches passed filter<commit_after>import sys
import os
sys.path.insert(0,os.path.dirname(sys.executable)+'/../lib/python2.7/site-packages/') #ignore other PYTHONPATH

import logging
import docopt
import schema
import brilws
import prettytable
import pandas as pd
import numpy as np
import ast
from brilws import api,params,display,formatter,lumiParameters,corrector
from brilws.cli import clicommonargs
import re,time, csv
from datetime import datetime
from sqlalchemy import *
import math
from itertools import izip_longest
from dateutil import tz
import pytz
log = logging.getLogger('brilws')
logformatter = logging.Formatter('%(levelname)s %(name)s %(message)s')
log.setLevel(logging.ERROR)
ch = logging.StreamHandler()
ch.setFormatter(logformatter)
log.addHandler(ch)

lumip = lumiParameters.ParametersObject()
lslengthsec= lumip.lslengthsec()
utctmzone = tz.gettz('UTC')
cerntmzone = tz.gettz('CEST')

class Unbuffered(object):
    def __init__(self, stream):
        self.stream = stream
    def write(self, data):
        self.stream.write(data)
        self.stream.flush()
    def __getattr__(self, attr):
        return getattr(self.stream.attr)
sys.stdout = Unbuffered(sys.stdout)

np.seterr(divide='ignore', invalid='ignore')

def xing_indexfilter(arr,constfactor=1.,xingMin=0.,xingTr=0.,xingId=[]):
    vidx = None    
    if xingTr:
        vidx = np.argwhere( np.logical_and( (arr*constfactor)>xingMin, arr>xingTr*np.max(arr) ) ).ravel()
    else:
        vidx = np.argwhere( (arr*constfactor)>xingMin ).ravel()
    if xingId :
        posidx = np.array(xingId)-1 #array position index is bunch index - 1
        return np.intersect1d(np.array(posidx),vidx)    
    return vidx

def totalprescaleNEW(hltprescval,l1seedlogic,l1prescvals):
    '''
    1) if ONE single seed, take it
    2) if OR:  (A.1 OR A.2 OR A.3 ...) ignore the triggers with prescale 0, and take the minimum of the remaining prescale values:
        if max(A.i) == 0
                A = 0
        else
                A = min(A.i | A.i > 0)
    3) if AND: A AND B AND C ... take the product of the prescale values A * B * C ...
    4) (optional)
       if (A.1 OR A.2 OR A.3 ...) AND (B.1 OR B.2 OR ...) AND ...
       - for each group, take the lowest non-zero prescale, or 0 if they are all zero
         if max(A.i) = 0
                A =
         else
                A = min(A.i | A.i > 0)
       - take the product of the prescales thus obtained:
         P = A * B * ...
       users still take their own risk and should use their own judgement.
    5)  if NOT, or a combination of AND/OR and NOT, reject
    '''
    totpresc = 0
    if not hltprescval or not l1prescvals or not l1seedlogic:
        return totpresc
    if np.all( np.array(l1prescvals)==1 ):
        totpresc = hltprescval        
    elif l1seedlogic=='ONE':
        totpresc = hltprescval*l1prescvals[0]
    elif l1seedlogic=='OR':
        if np.max( np.array(l1prescvals) ) == 0:
            return totpresc
        else:
            l = np.array(l1prescvals)
            l1p = np.min( l[l>0] )
        totpresc = hltprescval*l1p
    elif l1seedlogic=='AND':
        if np.max( np.array(l1prescvals) ) == 0:
            return totpresc
        l1p = np.prod( np.array(l1prescvals) )
        totpresc = hltprescval*l1p    
    return totpresc

    
def totalprescale(hltprescval,l1seedlogic,l1prescvals):
    totpresc = 0
    if not hltprescval or not l1prescvals or not l1seedlogic:
        return totpresc
    if np.all( np.array(l1prescvals)==1 ):
        totpresc = hltprescval
    elif l1seedlogic=='ONE':
        totpresc = hltprescval*l1prescvals[0]
    elif l1seedlogic=='OR':
        
        totpresc = hltprescval*np.min(l1prescvals)
    elif l1seedlogic=='AND':
        totpresc = hltprescval*np.max(l1prescvals)    
    return totpresc

def lumi_per_normtag(shards,lumiquerytype,dbengine,dbschema,runtot,datasource=None,normtag=None,withBX=False,byls=None,fh=None,csvwriter=None,ptable=None,scalefactor=1,totz=utctmzone,fillmin=None,fillmax=None,runmin=None,runmax=None,amodetagid=None,egev=None,beamstatusid=None,tssecmin=None,tssecmax=None,runlsSeries=None,hltl1map={},ignorel1mask=False,xingMin=0.,xingTr=0.,xingId=[],checkjson=False,datatagnameid=None):

    validitychecker = None
    lastvalidity = None
    if normtag and normtag is not 'withoutcorrection':
        normdata = api.iov_gettagdata(dbengine, normtag,schemaname=dbschema)
        if not normdata: raise ValueError('normtag %s does not exist'%normtag)
        validitychecker = ValidityChecker(normdata)
        
    for shard in shards:              
        lumiiter = None
        if lumiquerytype == 'bestresultonline':
            tablename = 'online_result_'+str(shard)
            shardexists = api.table_exists(dbengine,tablename,schemaname=dbschema)
            if not shardexists: continue
            rfields = ['fillnum','runnum','lsnum','timestampsec','cmson','beamstatusid','targetegev','delivered','recorded','avgpu','datasource','numbxbeamactive']
            if withBX: rfields = rfields+['bxdeliveredblob']                    
            lumiiter = api.online_resultIter(dbengine,tablename,schemaname=dbschema,fields=rfields,fillmin=fillmin,fillmax=fillmax,runmin=runmin,runmax=runmax,amodetagid=amodetagid,targetegev=egev,beamstatusid=beamstatusid,tssecmin=tssecmin,tssecmax=tssecmax,runlsselect=runlsSeries,sorted=True)
            
        elif lumiquerytype in ['detresultonline','detraw']:
            datatype = 'raw'
            if lumiquerytype=='detresultonline':
                datatype = 'result'
                
            tablename = datasource.lower()+'_'+datatype+'_'+str(shard)
            shardexists = api.table_exists(dbengine,tablename,schemaname=dbschema)
            if not shardexists: continue
            rfields = ['avglumi']
            idfields = ['fillnum','runnum','lsnum','timestampsec','beamstatusid','cmson','deadtimefrac','targetegev','numbxbeamactive']
            if withBX: rfields = rfields+['bxlumiblob']
            lumiiter = api.dataIter(dbengine,datasource,datatype,shard,datafields=rfields,idfields=idfields,schemaname=dbschema,fillmin=fillmin,fillmax=fillmax,runmin=runmin,runmax=runmax,amodetagid=amodetagid,targetegev=egev,beamstatusid=beamstatusid,tssecmin=tssecmin,tssecmax=tssecmax,runlsselect=runlsSeries,sorted=True,datatagnameid=datatagnameid)                    
                  
        if lumiiter is None:
            continue

        g_run_old = 0
        g_ls_trglastscaled_old = 0
        prescale_map = {} # for global scope
        g_hltconfigid = 0
        for row in lumiiter:
            fillnum = row['fillnum']
            runnum = row['runnum']
            lsnum = row['lsnum']            
            cmslsnum = lsnum
            timestampsec = row['timestampsec']
            cmson = row['cmson']
            if not cmson:
                cmslsnum = 0
            if hltl1map:
                if cmslsnum==0: continue  #cms is not running, skip.                
                if g_run_old!=runnum:     #on new run boundary, get hltconfigid
                    hltrunconfig = api.get_hltrunconfig(dbengine,runnum=runnum,schemaname=dbschema) #[['runnum','hltconfigid','hltkey']]
                    if not hltrunconfig:
                        continue
                    g_hltconfigid = [h[1] for h in hltrunconfig][0]
                    if not g_hltconfigid:
                        continue      
                    presc = api.get_prescidx_change(dbengine,runnum,schemaname=dbschema)#{runnum:[[lslastscaler,prescidx]] }
                    if presc.has_key(runnum):
                        presc = presc[runnum]
                this_prescidx = None
                if not presc:                    
                    ls_trglastscaled = 1
                else:
                    b = [ i[0] for i in presc if i[0]<=cmslsnum ] 
                    if not b:
                        ls_trglastscaled = 1
                        this_prescidx = None                        
                    else:
                        ls_trglastscaled = np.max( b )
                        this_prescidx = [t[1] for t in presc if t[0]==ls_trglastscaled][0]
                if (g_ls_trglastscaled_old != ls_trglastscaled) or (g_run_old != runnum): #on prescale change lumi section or new run
                    prescale_map = {} #clear prescaleindex change map on new prescale index value
                    if not hltl1map.has_key(g_hltconfigid):
                        continue
                    this_hltl1map = hltl1map[g_hltconfigid]#[[hltpathid,hltpathname,l1seedtype,l1seedbits]]
                    for grouped in this_hltl1map:
                        this_hltpathid = grouped[0]
                        hltpathname = grouped[1]
                        l1seedlogic = grouped[2]
                        l1candidates = grouped[3]
                        if this_prescidx is None:
                            hltprescval = 1
                            totpresc = 1
                        else:
                            hltprescval = api.get_hltprescale(dbengine,runnum,ls_trglastscaled,g_hltconfigid,this_prescidx,this_hltpathid,schemaname=dbschema)
                            rl = api.get_l1prescale(dbengine,runnum,ls_trglastscaled,l1candidates=l1candidates,prescidxs=this_prescidx,ignorel1mask=ignorel1mask ,schemaname=dbschema)
                            if not rl: continue
                            l1keys = [ k for k in rl.keys() if k[0]==this_prescidx ]
                            l1vals = [ rl[k] for k in l1keys ]                       
                            l1prescvals = [ v[0] for v in l1vals ]                            
                            totpresc = totalprescaleNEW(hltprescval,l1seedlogic,l1prescvals)
                            if not totpresc:
                                continue
                        prescale_map[hltpathname] = totpresc   
                    g_ls_trglastscaled_old = ls_trglastscaled
                g_run_old = runnum
            beamstatusid = row['beamstatusid']
            beamstatus = params._idtobeamstatus[beamstatusid]
            if beamstatus not in ['FLAT TOP','STABLE BEAMS','SQUEEZE','ADJUST']: continue
            tegev = row['targetegev']
            dtime = str(timestampsec)
            if totz is not None:
                d = datetime.fromtimestamp(int(timestampsec),tz=pytz.utc)
                dtime = d.astimezone(totz).strftime(params._datetimefm)
            delivered = recorded = avgpu = livefrac = 0.
            if checkjson:
                g_returnedls.append((runnum,lsnum))
            if lumiquerytype == 'bestresultonline': ##bestlumi
                if row.has_key('delivered') and row['delivered']:                    
                    delivered = np.true_divide(row['delivered']*lslengthsec,scalefactor)
                if delivered>0 and row.has_key('recorded') and row['recorded']:
                    recorded = np.true_divide(row['recorded']*lslengthsec,scalefactor)
                if delivered>0 and row.has_key('avgpu') and row['avgpu']:
                    avgpu = row['avgpu']
                ds = 'UNKNOWN' 
                if row.has_key('datasource') and row['datasource']:
                    ds = row['datasource']
                livefrac = np.true_divide(recorded,delivered)
                if withBX:    #--xing
                    bxlumi = None
                    bxlumistr = '[]'
                    if row.has_key('bxdeliveredblob'):
                        bxdeliveredarray = np.array(api.unpackBlobtoArray(row['bxdeliveredblob'],'f'))                        
                        totfactor = np.true_divide(lslengthsec,scalefactor)
                        bxidx = xing_indexfilter(bxdeliveredarray,constfactor=totfactor,xingMin=xingMin,xingTr=xingTr,xingId=xingId)
                        if bxidx is not None and bxidx.size>0:
                            bxdelivered = bxdeliveredarray[bxidx]*totfactor
                            bxlumi = np.transpose( np.array([bxidx+1,bxdelivered,bxdelivered*livefrac]) )
                        del bxdeliveredarray
                        del bxidx
                        if hltl1map: #--hltpath bx display
                            for pth in prescale_map.keys():                                
                                thispresc = prescale_map[pth]
                                if bxlumi is not None:
                                    a = map(formatter.bxlumi,bxlumi/thispresc)  
                                    bxlumistr = '['+' '.join(a)+']'                                
                                display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),ds,'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                                
                        else:        #normal bx display                   
                            if bxlumi is not None:
                                a = map(formatter.bxlumi,bxlumi)
                                bxlumistr = '['+' '.join(a)+']'                            
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),ds,'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    del bxlumi
                    
                elif byls:      #--byls
                    if hltl1map:#--hltpath display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum), dtime, pth, '%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),'%.1f'%(avgpu),ds] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:       #normal display
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),ds] , fh=fh, csvwriter=csvwriter, ptable=ptable)                        
            else:               ###lumisource
                if row.has_key('deadtimefrac') :
                    if row['deadtimefrac'] is not None:
                        livefrac = 1.-row['deadtimefrac']
                    else:
                        if cmslsnum!=0:
                            g_nulldeadtime.setdefault(runnum,[]).append(lsnum)
                        livefrac = 0.
                uncorrectedavglumi = row['avglumi'] #uncorrected, hz/ub
                avglumi = uncorrectedavglumi
                ncollidingbx = row['numbxbeamactive']
                uncorrectedbxlumi = None
                if row.has_key('bxlumiblob'):
                    uncorrectedbxlumi = np.array(api.unpackBlobtoArray(row['bxlumiblob'],'f'))
                bxlumi = uncorrectedbxlumi
                if validitychecker is not None:
                    if not lastvalidity or not validitychecker.isvalid(runnum,lastvalidity):
                        lastvalidity = validitychecker.getvalidity(runnum)
                    [normfunc,normparam] = validitychecker.getvaliddata(lastvalidity[0])
                    if withBX:
                        (avglumi,bxlumi) = corrector.applyCorrectionStr( normfunc,normparam, corrector.FunctionRoot(uncorrectedbxlumi,avglumi,ncollidingbx) )
                    else:
                        avglumi = corrector.applyCorrectionStr( normfunc,normparam, corrector.FunctionRoot(avglumi,avglumi,ncollidingbx) )[0]

                totfactor = np.true_divide(lslengthsec,scalefactor)
                delivered = avglumi*totfactor
                recorded = delivered*livefrac
                avgpu = lumip.avgpu( avglumi,ncollidingbx,g_minbias)                
                if withBX:      #--xing
                    bxlumistr = '[]'
                    if bxlumi is not None:
                        bxidx = xing_indexfilter(bxlumi,constfactor=totfactor,xingMin=xingMin,xingTr=xingTr,xingId=xingId)                        
                        if bxidx is not None and bxidx.size>0:                                      
                            bxdelivered =  bxlumi[bxidx]*totfactor
                            bxlumi = np.transpose( np.array([bxidx+1,bxdelivered,bxdelivered*livefrac]) )
                        else:
                            bxlumi = None                       
                        del uncorrectedbxlumi
                        del bxidx
                    if hltl1map:#--hltpath xing display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]                            
                            if bxlumi is not None:
                                a = map(formatter.bxlumi,bxlumi/thispresc)  
                                bxlumistr = '['+' '.join(a)+']'                            
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),datasource.upper(),'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:       #normal xing display
                        if bxlumi is not None:
                            a = map(formatter.bxlumi,bxlumi)                            
                            bxlumistr = '['+' '.join(a)+']'                        
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),datasource.upper(),'%s'%bxlumistr] , fh=fh, csvwriter=csvwriter, ptable=ptable)                        
                    del bxlumi
                elif byls:       #--byls
                    if hltl1map: #--hltpath display
                        for pth in prescale_map.keys():
                            thispresc = prescale_map[pth]                            
                            display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,pth,'%.3f'%(np.true_divide(delivered,thispresc)),'%.3f'%(np.true_divide(recorded,thispresc)),'%.1f'%(avgpu),datasource.upper()] , fh=fh, csvwriter=csvwriter, ptable=ptable)                            
                    else:        #normal display
                        display.add_row( ['%d:%d'%(runnum,fillnum),'%d:%d'%(lsnum,cmslsnum),dtime,beamstatus,'%d'%tegev,'%.3f'%(delivered),'%.3f'%(recorded),'%.1f'%(avgpu),datasource.upper()] , fh=fh, csvwriter=csvwriter, ptable=ptable)
            if not hltl1map:     #normal statistic collect
                if not runtot.has_key( ('',runnum) ):
                    runtot[('',runnum)] = {'fill':fillnum,'dtime':dtime,'nls':0,'ncms':0,'delivered':0,'recorded':0} 
{'fraction_non_alphanumeric': 0.10813226094727435, 'fraction_numerical': 0.00825316721863008, 'mean_word_length': 1.7871218225771006, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822010', 'n_tokens_mistral': 12430, 'n_tokens_neox': 11711, 'n_words': 1976}