<commit_before>import os
import re
import json
import logging


# Log object.
log = logging.getLogger(__name__)


# Axes information read from tables.
axes = {}


# Special files:
coord_file = "coordinate"


# Json file keys:
head_key = "header"
axis_key = "axis_entry"
var_key = "variable_entry"
freq_key = "frequency"
realm_key = "realm"
dims_key = "dimensions"
levs_key = "generic_levels"
cell_measures_key = "cell_measures"
cell_methods_key = "cell_methods"
valid_min_key = "valid_min"
valid_max_key = "valid_max"
cell_measure_axes = ["time","area","volume","latitude","longitude","depth"]


# Returns the axes defined for the input table.
def get_axis_info(table_id):
    global axes,coord_file
    result = axes.get(coord_file,{})
    overrides = axes.get(table_id,{})
    for k,v in overrides.iteritems():
        result[k] = v
    return result


# Class for cmor target objects, which represent output variables.
class cmor_target(object):
    def __init__(self,var_id__,tab_id__):
        self.variable = var_id__
        self.table = tab_id__
        self.dims = 2


# Derives the table id for the given file path
def get_table_id(filepath,prefix):
    fname = os.path.basename(filepath)
    regex = re.search("^" + prefix + "_.*.json$",fname)
    if(not regex):
        raise Exception("Unable to match file name",fname,"as cmor table json-file with prefix",prefix)
    return regex.group()[len(prefix) + 1:len(fname) - 5]


# Creates cmor-targets from the input json-file
def create_targets_for_file(filepath,prefix):
    global log,axes,head_key,freq_key,realm_key,levs_key,axis_key,var_key,dims_key,target_key
    global cell_methods_key,cell_measures_key,cell_measure_axes
    tabid = get_table_id(filepath,prefix)
    s = open(filepath).read()
    result = []
    try:
        data = json.loads(s)
    except ValueError as err:
        log.warning("Input table %s has been ignored. Reason: %s" % (filepath,format(err)))
        return result
    freq = None
    realm = None
    header = get_lowercase(data,head_key,None)
    modlevs = None
    if(header):
        freq = get_lowercase(header,freq_key,None)
        realm = get_lowercase(header,realm_key,None)
        modlevs = get_lowercase(header,levs_key,None)
    axes_entries = get_lowercase(data,axis_key,{})
    if(modlevs):
        for modlev in modlevs.split():
            axes_entries[modlev] = {"requested":"all"}
    axes[tabid] = axes_entries
    var_entries = get_lowercase(data,var_key,{})
    for k,v in var_entries.iteritems():
        target = cmor_target(k,tabid)
        target.frequency = freq
        target.realm = realm
        for k2,v2 in v.iteritems():
            key = k2.lower()
            setattr(target,key,v2)
            if(key == dims_key.lower()):
                spacedims = list(set([s for s in v2.split() if not s.lower().startswith("time")]) - set(["basin"]))
                target.dims = len(spacedims)
                zdims = list(set(spacedims)-set(["latitude","longitude"]))
                if(any(zdims)):
                    setattr(target,"z_dims",zdims)
            if(key in [cell_measures_key.lower(),cell_methods_key.lower()]):
                cell_measure_str = v2.strip()
                if(cell_measure_str not in ["@OPT","--OPT","",None]):
                    cell_measures = cell_measure_str.split(':')
                    for i in range(1,len(cell_measures)):
                        prev_words = cell_measures[i-1].split()
                        if(len(prev_words) == 0):
                            log.error("Error parsing cell measures %s for variable %s in table %s" % (v2,k,tabid))
                            break
                        measure_dim = prev_words[-1].strip().lower()
                        if(measure_dim not in cell_measure_axes):
                            log.error("Error parsing cell measures %s for variable %s in table %s" % (v2,k,tabid))
                            break
                        key = measure_dim + "_operator"<commit_msg>Fixed cell measures parsing error<commit_after>import os
import re
import json
import logging


# Log object.
log = logging.getLogger(__name__)


# Axes information read from tables.
axes = {}


# Special files:
coord_file = "coordinate"


# Json file keys:
head_key = "header"
axis_key = "axis_entry"
var_key = "variable_entry"
freq_key = "frequency"
realm_key = "realm"
dims_key = "dimensions"
levs_key = "generic_levels"
cell_measures_key = "cell_measures"
cell_methods_key = "cell_methods"
valid_min_key = "valid_min"
valid_max_key = "valid_max"
cell_measure_axes = ["time","area","volume","latitude","longitude","depth"]


# Returns the axes defined for the input table.
def get_axis_info(table_id):
    global axes,coord_file
    result = axes.get(coord_file,{})
    overrides = axes.get(table_id,{})
    for k,v in overrides.iteritems():
        result[k] = v
    return result


# Class for cmor target objects, which represent output variables.
class cmor_target(object):
    def __init__(self,var_id__,tab_id__):
        self.variable = var_id__
        self.table = tab_id__
        self.dims = 2


# Derives the table id for the given file path
def get_table_id(filepath,prefix):
    fname = os.path.basename(filepath)
    regex = re.search("^" + prefix + "_.*.json$",fname)
    if(not regex):
        raise Exception("Unable to match file name",fname,"as cmor table json-file with prefix",prefix)
    return regex.group()[len(prefix) + 1:len(fname) - 5]


# Creates cmor-targets from the input json-file
def create_targets_for_file(filepath,prefix):
    global log,axes,head_key,freq_key,realm_key,levs_key,axis_key,var_key,dims_key,target_key
    global cell_methods_key,cell_measures_key,cell_measure_axes
    tabid = get_table_id(filepath,prefix)
    s = open(filepath).read()
    result = []
    try:
        data = json.loads(s)
    except ValueError as err:
        log.warning("Input table %s has been ignored. Reason: %s" % (filepath,format(err)))
        return result
    freq = None
    realm = None
    header = get_lowercase(data,head_key,None)
    modlevs = None
    if(header):
        freq = get_lowercase(header,freq_key,None)
        realm = get_lowercase(header,realm_key,None)
        modlevs = get_lowercase(header,levs_key,None)
    axes_entries = get_lowercase(data,axis_key,{})
    if(modlevs):
        for modlev in modlevs.split():
            axes_entries[modlev] = {"requested":"all"}
    axes[tabid] = axes_entries
    var_entries = get_lowercase(data,var_key,{})
    for k,v in var_entries.iteritems():
        target = cmor_target(k,tabid)
        target.frequency = freq
        target.realm = realm
        for k2,v2 in v.iteritems():
            key = k2.lower()
            setattr(target,key,v2)
            if(key == dims_key.lower()):
                spacedims = list(set([s for s in v2.split() if not s.lower().startswith("time")]) - set(["basin"]))
                target.dims = len(spacedims)
                zdims = list(set(spacedims)-set(["latitude","longitude"]))
                if(any(zdims)):
                    setattr(target,"z_dims",zdims)
            if(key in [cell_measures_key.lower(),cell_methods_key.lower()]):
                cell_measure_str = re.sub("[\(\[].*?[\)\]]","",v2.strip())
                if(cell_measure_str not in ["@OPT","--OPT","",None]):
                    cell_measures = cell_measure_str.split(':')
                    for i in range(1,len(cell_measures)):
                        prev_words = cell_measures[i-1].split()
                        if(len(prev_words) == 0):
                            log.error("Error parsing cell measures %s for variable %s in table %s" % (v2,k,tabid))
                            break
                        measure_dim = prev_words[-1].strip().lower()
                        if(measure_dim not in cell_measure_axes):
                            log.error("Error parsing cell measures %s for variable %s in table %s" % (v2,k,tabid))
                            break
                        key = measure_dim + "_operator"
{'fraction_non_alphanumeric': 0.12070036011424314, 'fraction_numerical': 0.0037253197566124428, 'mean_word_length': 2.6459936622906293, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1492890', 'n_tokens_mistral': 2635, 'n_tokens_neox': 2546, 'n_words': 559}