<commit_before>import numpy as np
import pandas as pd
from sqlalchemy import exc,schema,types,Table,MetaData,Column
from datetime import datetime
import decimal
import os
import random
import time
#import yaml
import re
import contextlib
import sys
import ast
import logging
import string
from brilws import params
#from ConfigParser import SafeConfigParser

decimalcontext = decimal.getcontext().copy()
decimalcontext.prec = 3

log = logging.getLogger('brilws')

oracletypemap={
'uint8':'number(3)',
'uint16':'number(5)',
'uint32':'number(10)',
'uint64':'number(20)',
'int8':'number(3)',
'int16':'number(5)',
'int32':'number(10)',
'int64':'number(20)',
'float':'binary_float',
'double':'binary_double',
'string':'varchar2(4000)',
'blob':'blob',
'bool':'number(1)',
'timestamp':'timestamp'
}

sqlitetypemap={
'uint8':'INTEGER',
'uint16':'INTEGER',
'uint32':'INTEGER',
'uint64':'INTEGER',
'int8':'INTEGER',
'int16':'INTEGER',
'int32':'INTEGER',
'int64':'INTEGER',
'float':'REAL',
'double':'REAL',
'string':'TEXT',
'blob':'BLOB',
'bool':'INTEGER',
'timestamp':'DATETIME'
}

#def unpackBlobtoArray(iblob,itemtypecode):
#    '''
#    Inputs:
#    iblob: 
#    itemtypecode: python array type code 
#    '''
#    if itemtypecode not in ['c','b','B','u','h','H','i','I','l','L','f','d']:
#        raise RuntimeError('unsupported typecode '+itemtypecode)
#    result=array.array(itemtypecode)
#    blobstr=iblob.readline()
#    if not blobstr :
#        return None
#    result.fromstring(blobstr)
#    return result

    
####################
##    Selection API
####################
def parsecmsselectJSON(filepath_or_buffer,numpy=False):
    """
    parse cms selection json format
    input: 
        if file, parse file
        if string in dict format, eval as dict. Key is not required to be double quoted in this case
        if single string, convert to [int]
    output:
        pd.Series , index=runnum, value=[[lsmin,lsmax]]
    """
    d = get_filepath_or_buffer(filepath_or_buffer)

    try:
        result = pd.Series([int(d)])
        return result
    except ValueError:
        pass
    if os.path.isfile(d):
        result = pd.read_json(d,orient='index',convert_axes=False,typ='Series',numpy=numpy)
    else:
        d = ast.literal_eval(d)
        result = pd.Series(d)
    result.index = [int(i) for i in result.index]
    return result

@contextlib.contextmanager<commit_msg>add parse iovtagselectJSON<commit_after>import numpy as np
import pandas as pd
from sqlalchemy import exc,schema,types,Table,MetaData,Column
from datetime import datetime
import decimal
import os
import random
import time
#import yaml
import re
import contextlib
import sys
import ast
import logging
import string
from brilws import params
#from ConfigParser import SafeConfigParser

decimalcontext = decimal.getcontext().copy()
decimalcontext.prec = 3

log = logging.getLogger('brilws')

oracletypemap={
'uint8':'number(3)',
'uint16':'number(5)',
'uint32':'number(10)',
'uint64':'number(20)',
'int8':'number(3)',
'int16':'number(5)',
'int32':'number(10)',
'int64':'number(20)',
'float':'binary_float',
'double':'binary_double',
'string':'varchar2(4000)',
'blob':'blob',
'bool':'number(1)',
'timestamp':'timestamp'
}

sqlitetypemap={
'uint8':'INTEGER',
'uint16':'INTEGER',
'uint32':'INTEGER',
'uint64':'INTEGER',
'int8':'INTEGER',
'int16':'INTEGER',
'int32':'INTEGER',
'int64':'INTEGER',
'float':'REAL',
'double':'REAL',
'string':'TEXT',
'blob':'BLOB',
'bool':'INTEGER',
'timestamp':'DATETIME'
}

#def unpackBlobtoArray(iblob,itemtypecode):
#    '''
#    Inputs:
#    iblob: 
#    itemtypecode: python array type code 
#    '''
#    if itemtypecode not in ['c','b','B','u','h','H','i','I','l','L','f','d']:
#        raise RuntimeError('unsupported typecode '+itemtypecode)
#    result=array.array(itemtypecode)
#    blobstr=iblob.readline()
#    if not blobstr :
#        return None
#    result.fromstring(blobstr)
#    return result

    
####################
##    Selection API
####################

_maxls = 9999
_maxrun = 999999

def parseselectionJSON(filepath_or_buffer):
    d = get_filepath_or_buffer(filepath_or_buffer)
    data = ''
    if os.path.isfile(d):
        with open(d,'r') as f:
            data = f.read().lstrip()        
    else:
        data = filepath_or_buffer.lstrip()
    if data[0]=='[':
        return parseiovtagselectionJSON(data)
    else:
        return parsecmsselectJSON(data)
        
def parseiovtagselectionJSON(filepath_or_buffer):
    """
    parse iov tag selection file
    input:
        if file, parse file
    output:
        list [iovtag,"{run:[[1,9999]],run:[[1,9999]]}" , [iovtag,"{run:[[lsstart,lsstop]],...}" ]                     
    """
    result = None
    d = get_filepath_or_buffer(filepath_or_buffer)    
    if os.path.isfile(filepath_or_buffer):
        result = pd.read_json(d,orient='index',convert_axes=False,typ='Series')
    else:
        spacer = re.compile(r'^\s+')
        d = spacer.sub('',d)
        word = re.compile(r'(\w*[A-Za-z]\w*),')
        d = ast.literal_eval(d)
        result = pd.Series(d)
    final = []
    for r in result:
        iovtag = r[0]
        payload = r[1:]
        for piece in payload :
            if isinstance(piece,dict):
                final.append([iovtag,str(piece)])
            else:
                p = '{"%s":[[1,%s]]}'%(piece,_maxls)
                final.append([iovtag,p])

    return final

def parsecmsselectJSON(filepath_or_buffer,numpy=False):
    """
    parse cms selection json format
    input: 
        if file, parse file
        if string in dict format, eval as dict. Key is not required to be double quoted in this case
        if single string, convert to [int]
    output:
        pd.Series , index=runnum, value=[[lsmin,lsmax]]
    """
    d = get_filepath_or_buffer(filepath_or_buffer)

    try:
        result = pd.Series([int(d)])
        return result
    except ValueError:
        pass

    if os.path.isfile(d):
        result = pd.read_json(d,orient='index',convert_axes=False,typ='Series',numpy=numpy)
    else:
        d = ast.literal_eval(d)
        result = pd.Series(d)
    result.index = [int(i) for i in result.index]
    return result

@contextlib.contextmanager
{'fraction_non_alphanumeric': 0.1700789940351443, 'fraction_numerical': 0.0190230533612768, 'mean_word_length': 3.743119266055046, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7339016', 'n_tokens_mistral': 2204, 'n_tokens_neox': 2085, 'n_words': 447}