<commit_before>import string
import utils_c4 as utils
import os
import os.path as op
import shutil, collections
from versionConfig import version, versionComment

NT_project = collections.namedtuple( 'project', ['id','v'] )
NT_fnParts = collections.namedtuple( 'fnParts', ['len','fxLen','unfLen','checkTLen','ixDomain','ixFreq'] )

##############################################################################
# Configure config-file paths
#
# All configuration directories, e.g. cmip5_vocabs, are looked for in a single
# parent directory.  This is the "config" directory within the package unless 
# the environment variable CC_CONFIG_DIR is set.

HERE = op.dirname(__file__)
CC_CONFIG_DEFAULT_DIR = op.join(HERE, 'config')
CC_CONFIG_DIR = os.environ.get('CC_CONFIG_DIR', CC_CONFIG_DEFAULT_DIR)

##############################################################################

validCmip5Experiments = ['1pctCO2', 'abrupt4xCO2', 'amip', 'amip4K', 'amip4xCO2', 'amipFuture', 'aqua4K', 'aqua4xCO2', 'aquaControl', 'decadal1959', 'decadal1960', 'decadal1961', 'decadal1962', 'decadal1963', 'decadal1964', 'decadal1965', 'decadal1966', 'decadal1967', 'decadal1968', 'decadal1969', 'decadal1970', 'decadal1971', 'decadal1972', 'decadal1973', 'decadal1974', 'decadal1975', 'decadal1976', 'decadal1977', 'decadal1978', 'decadal1979', 'decadal1980', 'decadal1981', 'decadal1982', 'decadal1983', 'decadal1984', 'decadal1985', 'decadal1986', 'decadal1987', 'decadal1988', 'decadal1989', 'decadal1990', 'decadal1991', 'decadal1992', 'decadal1993', 'decadal1994', 'decadal1995', 'decadal1996', 'decadal1997', 'decadal1998', 'decadal1999', 'decadal2000', 'decadal2001', 'decadal2002', 'decadal2003', 'decadal2004', 'decadal2005', 'decadal2006', 'decadal2007', 'decadal2008', 'decadal2009', 'decadal2010', 'decadal2011', 'decadal2012', 'esmControl', 'esmFdbk1', 'esmFdbk2', 'esmFixClim1', 'esmFixClim2', 'esmHistorical', 'esmrcp85', 'historical', 'historicalExt', 'historicalGHG', 'historicalMisc', 'historicalNat', 'lgm', 'midHolocene', 'noVolc1960', 'noVolc1965', 'noVolc1970', 'noVolc1975', 'noVolc1980', 'noVolc1985', 'noVolc1990', 'noVolc1995', 'noVolc2000', 'noVolc2005', 'past1000', 'piControl', 'rcp26', 'rcp45', 'rcp60', 'rcp85', 'sst2020', 'sst2030', 'sst2090', 'sst2090rcp45', 'sstClim', 'sstClim4xCO2', 'sstClimAerosol', 'sstClimSulfate', 'volcIn2010']

validCordexExperiment = validCmip5Experiments + ['evaluation']


validCmip5Frequencies = ['fx','yr','monClim','mon','day','6hr','3hr','subhr']
validCordexFrequencies = ['fx','sem','mon','day','6hr','3hr']
validSpecsFrequencies = ['fx','mon','day','6hr']
validCcmiFrequencies = ['fx','yr','mon','day','hr','subhr']
validSpecsExptFamilies = map( lambda x: string.strip( x ), 
                              open( op.join(CC_CONFIG_DIR, 'specs_vocabs/exptFamily.txt' )).readlines() )

validCordexDomainsL = [ 'SAM-44', 'CAM-44', 'NAM-44', 'EUR-44', 'AFR-44', 'WAS-44', 'EAS-44', 'CAS-44', 'AUS-44', 'ANT-44', 'ARC-44', 'MED-44']
validCordexDomainsLi = map( lambda x: x + 'i', validCordexDomainsL )
validCordexDomainsH = ['EUR-11']
validCordexDomains = validCordexDomainsL + validCordexDomainsLi + validCordexDomainsH

plevRequired = ['clh', 'clm', 'cll', 'ua850', 'va850', 'ta850', 'hus850', 'ua500', 'va500', 'ta500', 'zg500', 'ua200', 'va200', 'ta200', 'zg200']
plevBndsRequired = ['clh', 'clm', 'cll']
heightRequired = ['tas','tasmax','tasmin','huss','sfcWind','sfcWindmax','wsgsmax','uas','vas']


ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/GCMModelName.txt' )).readlines()
validGcmNames = []
for l in ii:
  if l[0] != '#' and len( string.strip(l) ) > 0:
    validGcmNames.append( string.split(l)[0] )

ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/RCMModelName.txt' )).readlines()
validRcmNames = []
validInstNames = []
for l in ii:
  if l[0] != '#' and len( string.strip(l) ) > 0:
    bits = string.split(l)
    validRcmNames.append( bits[0] )
    validInstNames.append( bits[1] )

plevValues = {'clh':22000, 'clm':56000, 'cll':84000}
for i in [200,500,850]:
  for v in ['zg','ua', 'va', 'ta', 'hus']:
    k = '%s%s' % (v,i)
    plevValues[k] = i*100

heightRequired = ['tas', 'tasmax', 'tasmin', 'huss', 'sfcWind', 'sfcWindmax', 'wsgsmax', 'uas', 'vas']
heightValues = {}
heightRange = {}
for v in heightRequired:
  if v in ['tas', 'tasmax', 'tasmin', 'huss']:
    heightValues[v] = 2
  else:
    heightValues[v] = 10
  heightRange[v] = (1.5,10.)

ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/cordex_domains.csv' )).readlines()
keys = ['name','tag','res','grid_np_lon','grid_np_lat','nlon','nlat','w','e','s','n']
rotatedPoleGrids = {}
for l in ii[2:16]:
  bits = string.split(string.strip(l),',')
  ee = {}
  i = 0
  for k in keys:
    if k in ['nlon','nlat']:
      ee[k] = int(bits[i])
    elif k in ['grid_np_lon','grid_np_lat','w','e','s','n','res']:
      if bits[i] != 'N/A':
        ee[k] = float(bits[i])
      else:
        ee[k] = bits[i]
    else:
      ee[k] = bits[i]
    i += 1
    rotatedPoleGrids[bits[1]] = ee

##Area,Name, deg,Nlon,Nlat,West8,East8,South8,North8,
keys = ['name','tag','res','nlon','nlat','w','e','s','n']
interpolatedGrids = {}
for l in ii[18:33]:
  bits = string.split(string.strip(l),',')
  ee = {}
  i = 0
  for k in keys:
    if k in ['nlon','nlat']:
      ee[k] = int(bits[i])
    elif k in ['w','e','s','n','res']:
        ee[k] = float(bits[i])
    else:
      ee[k] = bits[i]
    i += 1
    interpolatedGrids[bits[1]] = ee

class readVocab(object):

  def __init__(self,dir):
    self.dir = dir

  def driver(self,tt):
    if tt[0] == "simpleList":
      tx = tt + (None,None,None,None)
      return self.getSimpleList( tx[1],bit=tx[2],omt=tx[3],options=tx[4] )
    else:
      print 'readVocab.driver: option %s not recgnised' % tt[0]

  def getEvalAssign(self, file ):
    ii = open('%s/%s/%s' % (CC_CONFIG_DIR, self.dir,file) )
    ll = []
    for l in ii.readlines():
      if l[0] != '#':
        ll.append( l )
    assert len(ll) == 1, 'Too many lines in %s' % file 
    try:
      return eval( ll[0] )
    except:
      print 'Failed to evaluate configuration line from %s:\n%s' % (file,l[0])
      raise

  def getSimpleList(self,file,bit=None,omt=None,options=None):
    ii = open('%s/%s/%s' % (CC_CONFIG_DIR, self.dir,file) )
    oo = []
    if options == None:
      olist = []
    else:
      olist = string.split(options,',')

    if 'returnMappings' in olist:
      assert bit in [0, -1], 'only support returnMappings for bit = -1 or 0'
      ee = {}

    for l in ii.readlines():
      if l[0] != '#':
        ll = string.strip(l)
        if omt == 'last':
          oo.append(string.join(string.split(ll)[:-1]))
        elif bit == None:
          oo.append(ll)
        else:
          if 'returnMappings' in olist:
            bb = string.split(ll)
            if bit == -1:
              ee[bb[-1]] = string.join( bb[:-1] )
            else:
              ee[bb[0]] = string.join( bb[1:] )
            oo.append( bb[bit] )
          else:
            oo.append(string.split(ll)[bit])

    if 'noneMap' in olist:
      oo = map( lambda x: {'None':None}.get(x,x), oo )

    if 'returnMappings' in olist:
      return oo, ee
    else:
      return oo

validSpecsInstitutions = ['IC3', 'MPI-M', 'KNMI', 'UOXF', 'CNRM-CERFACS', 'ENEA', 'MOHC', 'SMHI', 'IPSL', 'UREAD', 'ECWMF']

class projectConfig(object):

  def __init__(self, project, version=-1):
    knownProjects = ['CMIP5','CCMI','CORDEX','SPECS','ESA-CCI', '__dummy']
    assert project in knownProjects, 'Project %s not in knownProjects %s' % (project, str(knownProjects))

    self.project = project
    self.fNameSep = '_'
    self.varIndex = 0
    self.fnvdict = None
    self.varTables='CMIP'
    self.checkVarType = True
    self.projectV = NT_project(project,version)
    self.gridSpecTol = 0.01
## default encoding of time range in file names: YYYY[MM[DD[HH]]]-YYYY[MM[DD[HH]]]
    self.trangeType = 'CMIP'
    self.controlledFnParts = []
    if project == 'CORDEX':
      self.requiredGlobalAttributes = [ 'institute_id', 'contact', 'rcm_version_id', 'product', 'CORDEX_domain', 'creation_date', \
             'frequency', 'model_id', 'driving_model_id', 'driving_experiment', 'driving_model_ensemble_member', 'experiment_id']
      self.controlledGlobalAttributes = ['frequency', 'driving_experiment_name', 'project_id', 'CORDEX_domain', 'driving_model_id', 'model_id', 'institute_id','driving_model_ensemble_member','rcm_version_id']
      self.globalAttributesInFn = [None,'CORDEX_domain','driving_model_id','experiment_id','driving_model_ensemble_member','model_id','rcm_version_id']
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'@var','institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'driving_model_ensemble_member', 'model':'model_id', 'driving_model':'driving_model_id', \
                        'frequency':'frequency', \
                        'project':'project_id', 'domain':'CORDEX_domain', 'model_version':'rcm_version_id' }

    elif project == 'SPECS':
      lrdr = readVocab( 'specs_vocabs/')
      self.requiredGlobalAttributes = lrdr.driver( ('simpleList', 'globalAts.txt' ) )
      self.exptFamilies = lrdr.driver( ('simpleList', 'exptFamily.txt', 0 ) )
      self.controlledGlobalAttributes = lrdr.driver( ('simpleList', 'controlledGlobalAttributes_sv0101.txt' ) )
                       ##'initialization_method','physics_version','realization']
      self.globalAttributesInFn = lrdr.driver( ('simpleList', 'globalAttributesInFn_sv0101.txt',0,0,'noneMap' ) )
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      oo, self.drsMappings = lrdr.driver( ('simpleList', 'drsMappings_sv0101.txt',0,0,'returnMappings' ) )

      ##self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        ##'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        ##'frequency':'frequency', 'start_date':'@forecast_reference_time', \
                        ##'table':'@mip_id',
                        ##'project':'project_id'}

    elif project == 'CMIP5':
      lrdr = readVocab( 'cmip5_vocabs/')
      self.requiredGlobalAttributes = [ 'contact', 'product', 'creation_date', 'tracking_id', \
              'experiment_id']
      ##self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt' )
      self.controlledGlobalAttributes = [ 'project_id','experiment_id', 'frequency','Conventions','modeling_realm', \
                       'initialization_method','physics_version','realization']
      self.globalAttributesInFn = [None,'@mip_id','model_id','experiment_id','@ensemble']
#sic_Oimon_EC-Earth2_seaIceBestInit_S19910501_series1_r1i1p1_199501-199502.nc 
## mip_id derived from global attribute Table_id (CMOR convention); experiment family derived from experiment_id, ensemble derived from rip attributes.
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
## key: DRS element name, value: global attribute name or tag for mapping from file information ("@....").
      self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        'frequency':'frequency',  'table':'@mip_id',
                        'project':'project_id'}

    elif project == 'CCMI':
      lrdr = readVocab( 'ccmi_vocabs/')
      self.requiredGlobalAttributes = [ 'creation_date', 'tracking_id', 'forcing', 'model_id', 'parent_experiment_id', 'parent_experiment_rip', 'branch_time', 'contact', 'institute_id' ]
      self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt', bit=0 )
      self.controlledGlobalAttributes = [ 'experiment_id', 'project', 'frequency' ]
      self.globalAttributesInFn = [None,'@mip_id','model_id','experiment_id','@ensemble']
      self.requiredVarAttributes = ['long_name', 'units']
      self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        'frequency':'frequency',  'table':'@mip_id',
                        'project':'project_id'}

    elif project == 'ESA-CCI':
      lrdr = readVocab( 'esacci_vocabs/')
      self.varTables='FLAT'
      self.fNameSep = '-'
      self.checkVarType = False
      self.requiredGlobalAttributes = lrdr.getSimpleList( 'requiredGlobalAts.txt', bit=0 )
      self.controlledGlobalAttributes = ['platform','sensor','project','Conventions','institution','cdm_data_type','time_coverage_duration','spatial_resolution' ]
      self.controlledFnParts = ['level','cciProject','var','version']
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'#var','platform':'platform','sensor':'sensor','level':'#level', \
                'standard_name':'*standard_name', \
                'algorithm':'$algorithm:unset', 'frequency':'$frequency', \
                'spatial_resolution':'spatial_resolution', 'ecv':'@ecv','version':'#version','convention_version':'#gdsv'}
      self.globalAttributesInFn = [None,]
    elif project == '__dummy':
      self.requiredGlobalAttributes = map( lambda x: 'ga%s' % x, range(10) )
      self.controlledGlobalAttributes = [ ]
      self.globalAttributesInFn = [None,'ga2', 'ga3', 'ga4' ]
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'@var'}

####### used in checkStandardDims

    self.plevRequired = plevRequired
    self.plevValues = plevValues
    self.heightRequired = heightRequired
    self.heightValues = heightValues
    self.heightRange = heightRange

####### used in checkGrids
    self.rotatedPoleGrids = rotatedPoleGrids
    self.interpolatedGrids = interpolatedGrids
    self.doCheckGrids = self.projectV.id in ['CORDEX',]

####### used in checkFileName (freqIndex also used in checkByVar)

    if self.projectV.id == 'CORDEX':
      self.fnParts = NT_fnParts( len=[8,9], fxLen=[8,],  unfLen=[9,], checkTLen=True, ixDomain=1, ixFreq=7 )
    elif self.projectV.id == 'CMIP5':
      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None )
    elif self.projectV.id == 'SPECS':
      self.fnParts = lrdr.getEvalAssign( 'fnParts_sv0101.txt' )
     ## self.fnParts = NT_fnParts( len=[6,7], fxLen=[6,],  unfLen=[7,], checkTLen=False, ixDomain=None, ixFreq=1 )
      print self.fnParts
      
    elif self.projectV.id == 'CCMI':
      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None )
    elif self.projectV.id == 'ESA-CCI':
      self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=1 )
      self.trangeType = 'ESA-CCI'
    elif self.projectV.id == '__dummy':
      self.fnParts = NT_fnParts( len=[4,5], fxLen=[4,],  unfLen=[5,], checkTLen=False, ixDomain=None, ixFreq=1 )
<commit_msg>tidied up vocab handling<commit_after>import string
import utils_c4 as utils
import os
import os.path as op
import shutil, collections
from versionConfig import version, versionComment

NT_project = collections.namedtuple( 'project', ['id','v'] )
NT_fnParts = collections.namedtuple( 'fnParts', ['len','fxLen','unfLen','checkTLen','ixDomain','ixFreq'] )

##############################################################################
# Configure config-file paths
#
# All configuration directories, e.g. cmip5_vocabs, are looked for in a single
# parent directory.  This is the "config" directory within the package unless 
# the environment variable CC_CONFIG_DIR is set.

HERE = op.dirname(__file__)
CC_CONFIG_DEFAULT_DIR = op.join(HERE, 'config')
CC_CONFIG_DIR = os.environ.get('CC_CONFIG_DIR', CC_CONFIG_DEFAULT_DIR)

##############################################################################

validCmip5Experiments = ['1pctCO2', 'abrupt4xCO2', 'amip', 'amip4K', 'amip4xCO2', 'amipFuture', 'aqua4K', 'aqua4xCO2', 'aquaControl', 'decadal1959', 'decadal1960', 'decadal1961', 'decadal1962', 'decadal1963', 'decadal1964', 'decadal1965', 'decadal1966', 'decadal1967', 'decadal1968', 'decadal1969', 'decadal1970', 'decadal1971', 'decadal1972', 'decadal1973', 'decadal1974', 'decadal1975', 'decadal1976', 'decadal1977', 'decadal1978', 'decadal1979', 'decadal1980', 'decadal1981', 'decadal1982', 'decadal1983', 'decadal1984', 'decadal1985', 'decadal1986', 'decadal1987', 'decadal1988', 'decadal1989', 'decadal1990', 'decadal1991', 'decadal1992', 'decadal1993', 'decadal1994', 'decadal1995', 'decadal1996', 'decadal1997', 'decadal1998', 'decadal1999', 'decadal2000', 'decadal2001', 'decadal2002', 'decadal2003', 'decadal2004', 'decadal2005', 'decadal2006', 'decadal2007', 'decadal2008', 'decadal2009', 'decadal2010', 'decadal2011', 'decadal2012', 'esmControl', 'esmFdbk1', 'esmFdbk2', 'esmFixClim1', 'esmFixClim2', 'esmHistorical', 'esmrcp85', 'historical', 'historicalExt', 'historicalGHG', 'historicalMisc', 'historicalNat', 'lgm', 'midHolocene', 'noVolc1960', 'noVolc1965', 'noVolc1970', 'noVolc1975', 'noVolc1980', 'noVolc1985', 'noVolc1990', 'noVolc1995', 'noVolc2000', 'noVolc2005', 'past1000', 'piControl', 'rcp26', 'rcp45', 'rcp60', 'rcp85', 'sst2020', 'sst2030', 'sst2090', 'sst2090rcp45', 'sstClim', 'sstClim4xCO2', 'sstClimAerosol', 'sstClimSulfate', 'volcIn2010']

validCordexExperiment = validCmip5Experiments + ['evaluation']


validCmip5Frequencies = ['fx','yr','monClim','mon','day','6hr','3hr','subhr']
validCordexFrequencies = ['fx','sem','mon','day','6hr','3hr']
validSpecsFrequencies = ['fx','mon','day','6hr']
validCcmiFrequencies = ['fx','yr','mon','day','hr','subhr']
validSpecsExptFamilies = map( lambda x: string.strip( x ), 
                              open( op.join(CC_CONFIG_DIR, 'specs_vocabs/exptFamily.txt' )).readlines() )

validCordexDomainsL = [ 'SAM-44', 'CAM-44', 'NAM-44', 'EUR-44', 'AFR-44', 'WAS-44', 'EAS-44', 'CAS-44', 'AUS-44', 'ANT-44', 'ARC-44', 'MED-44']
validCordexDomainsLi = map( lambda x: x + 'i', validCordexDomainsL )
validCordexDomainsH = ['EUR-11']
validCordexDomains = validCordexDomainsL + validCordexDomainsLi + validCordexDomainsH

plevRequired = ['clh', 'clm', 'cll', 'ua850', 'va850', 'ta850', 'hus850', 'ua500', 'va500', 'ta500', 'zg500', 'ua200', 'va200', 'ta200', 'zg200']
plevBndsRequired = ['clh', 'clm', 'cll']
heightRequired = ['tas','tasmax','tasmin','huss','sfcWind','sfcWindmax','wsgsmax','uas','vas']


ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/GCMModelName.txt' )).readlines()
validGcmNames = []
for l in ii:
  if l[0] != '#' and len( string.strip(l) ) > 0:
    validGcmNames.append( string.split(l)[0] )

ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/RCMModelName.txt' )).readlines()
validRcmNames = []
validInstNames = []
for l in ii:
  if l[0] != '#' and len( string.strip(l) ) > 0:
    bits = string.split(l)
    validRcmNames.append( bits[0] )
    validInstNames.append( bits[1] )

plevValues = {'clh':22000, 'clm':56000, 'cll':84000}
for i in [200,500,850]:
  for v in ['zg','ua', 'va', 'ta', 'hus']:
    k = '%s%s' % (v,i)
    plevValues[k] = i*100

heightRequired = ['tas', 'tasmax', 'tasmin', 'huss', 'sfcWind', 'sfcWindmax', 'wsgsmax', 'uas', 'vas']
heightValues = {}
heightRange = {}
for v in heightRequired:
  if v in ['tas', 'tasmax', 'tasmin', 'huss']:
    heightValues[v] = 2
  else:
    heightValues[v] = 10
  heightRange[v] = (1.5,10.)

ii = open( op.join(CC_CONFIG_DIR, 'cordex_vocabs/cordex_domains.csv' )).readlines()
keys = ['name','tag','res','grid_np_lon','grid_np_lat','nlon','nlat','w','e','s','n']
rotatedPoleGrids = {}
for l in ii[2:16]:
  bits = string.split(string.strip(l),',')
  ee = {}
  i = 0
  for k in keys:
    if k in ['nlon','nlat']:
      ee[k] = int(bits[i])
    elif k in ['grid_np_lon','grid_np_lat','w','e','s','n','res']:
      if bits[i] != 'N/A':
        ee[k] = float(bits[i])
      else:
        ee[k] = bits[i]
    else:
      ee[k] = bits[i]
    i += 1
    rotatedPoleGrids[bits[1]] = ee

##Area,Name, deg,Nlon,Nlat,West8,East8,South8,North8,
keys = ['name','tag','res','nlon','nlat','w','e','s','n']
interpolatedGrids = {}
for l in ii[18:33]:
  bits = string.split(string.strip(l),',')
  ee = {}
  i = 0
  for k in keys:
    if k in ['nlon','nlat']:
      ee[k] = int(bits[i])
    elif k in ['w','e','s','n','res']:
        ee[k] = float(bits[i])
    else:
      ee[k] = bits[i]
    i += 1
    interpolatedGrids[bits[1]] = ee

class readVocab(object):

  def __init__(self,dir):
    self.dir = dir

  def driver(self,tt):
    if tt[0] == "simpleList":
      tx = tt + (None,None,None,None)
      return self.getSimpleList( tx[1],bit=tx[2],omt=tx[3],options=tx[4] )
    else:
      print 'readVocab.driver: option %s not recgnised' % tt[0]

  def getEvalAssign(self, file ):
    ii = open('%s/%s/%s' % (CC_CONFIG_DIR, self.dir,file) )
    ll = []
    for l in ii.readlines():
      if l[0] != '#':
        ll.append( l )
    assert len(ll) == 1, 'Too many lines in %s' % file 
    try:
      return eval( ll[0] )
    except:
      print 'Failed to evaluate configuration line from %s:\n%s' % (file,l[0])
      raise

  def getSimpleList(self,file,bit=None,omt=None,options=None):
    ii = open('%s/%s/%s' % (CC_CONFIG_DIR, self.dir,file) )
    oo = []
    if options == None:
      olist = []
    else:
      olist = string.split(options,',')

    if 'returnMappings' in olist:
      assert bit in [0, -1], 'only support returnMappings for bit = -1 or 0'
      ee = {}

    for l in ii.readlines():
      if l[0] != '#':
        ll = string.strip(l)
        if omt == 'last':
          oo.append(string.join(string.split(ll)[:-1]))
        elif bit == None:
          oo.append(ll)
        else:
          if 'returnMappings' in olist:
            bb = string.split(ll)
            if bit == -1:
              ee[bb[-1]] = string.join( bb[:-1] )
            else:
              ee[bb[0]] = string.join( bb[1:] )
            oo.append( bb[bit] )
          else:
            oo.append(string.split(ll)[bit])

    if 'noneMap' in olist:
      oo = map( lambda x: {'None':None}.get(x,x), oo )

    if 'returnMappings' in olist:
      return oo, ee
    else:
      return oo

validSpecsInstitutions = ['IC3', 'MPI-M', 'KNMI', 'UOXF', 'CNRM-CERFACS', 'ENEA', 'MOHC', 'SMHI', 'IPSL', 'UREAD', 'ECWMF']

class projectConfig(object):

  def __init__(self, project, version=-1):
    knownProjects = ['CMIP5','CCMI','CORDEX','SPECS','ESA-CCI', '__dummy']
    assert project in knownProjects, 'Project %s not in knownProjects %s' % (project, str(knownProjects))

    self.project = project
    self.fNameSep = '_'
    self.varIndex = 0
    self.fnvdict = None
    self.varTables='CMIP'
    self.checkVarType = True
    self.projectV = NT_project(project,version)
    self.gridSpecTol = 0.01
## default encoding of time range in file names: YYYY[MM[DD[HH]]]-YYYY[MM[DD[HH]]]
    self.trangeType = 'CMIP'
    self.controlledFnParts = []
    if project == 'CORDEX':
      self.requiredGlobalAttributes = [ 'institute_id', 'contact', 'rcm_version_id', 'product', 'CORDEX_domain', 'creation_date', \
             'frequency', 'model_id', 'driving_model_id', 'driving_experiment', 'driving_model_ensemble_member', 'experiment_id']
      self.controlledGlobalAttributes = ['frequency', 'driving_experiment_name', 'project_id', 'CORDEX_domain', 'driving_model_id', 'model_id', 'institute_id','driving_model_ensemble_member','rcm_version_id']
      self.globalAttributesInFn = [None,'CORDEX_domain','driving_model_id','experiment_id','driving_model_ensemble_member','model_id','rcm_version_id']
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'@var','institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'driving_model_ensemble_member', 'model':'model_id', 'driving_model':'driving_model_id', \
                        'frequency':'frequency', \
                        'project':'project_id', 'domain':'CORDEX_domain', 'model_version':'rcm_version_id' }

    elif project == 'SPECS':
      lrdr = readVocab( 'specs_vocabs/')
      self.requiredGlobalAttributes = lrdr.driver( ('simpleList', 'globalAts.txt' ) )
      self.exptFamilies = lrdr.driver( ('simpleList', 'exptFamily.txt', 0 ) )
      self.controlledGlobalAttributes = lrdr.driver( ('simpleList', 'controlledGlobalAttributes_sv0101.txt' ) )
                       ##'initialization_method','physics_version','realization']
      self.globalAttributesInFn = lrdr.driver( ('simpleList', 'globalAttributesInFn_sv0101.txt',0,0,'noneMap' ) )
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      oo, self.drsMappings = lrdr.driver( ('simpleList', 'drsMappings_sv0101.txt',0,0,'returnMappings' ) )

      ##self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        ##'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        ##'frequency':'frequency', 'start_date':'@forecast_reference_time', \
                        ##'table':'@mip_id',
                        ##'project':'project_id'}

    elif project == 'CMIP5':
      lrdr = readVocab( 'cmip5_vocabs/')
      self.requiredGlobalAttributes = [ 'contact', 'product', 'creation_date', 'tracking_id', \
              'experiment_id']
      ##self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt' )
      self.controlledGlobalAttributes = [ 'project_id','experiment_id', 'frequency','Conventions','modeling_realm', \
                       'initialization_method','physics_version','realization']
      self.globalAttributesInFn = [None,'@mip_id','model_id','experiment_id','@ensemble']
#sic_Oimon_EC-Earth2_seaIceBestInit_S19910501_series1_r1i1p1_199501-199502.nc 
## mip_id derived from global attribute Table_id (CMOR convention); experiment family derived from experiment_id, ensemble derived from rip attributes.
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
## key: DRS element name, value: global attribute name or tag for mapping from file information ("@....").
      self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        'frequency':'frequency',  'table':'@mip_id',
                        'project':'project_id'}

    elif project == 'CCMI':
      lrdr = readVocab( 'ccmi_vocabs/')
      self.requiredGlobalAttributes = [ 'creation_date', 'tracking_id', 'forcing', 'model_id', 'parent_experiment_id', 'parent_experiment_rip', 'branch_time', 'contact', 'institute_id' ]
      self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt', bit=0 )
      self.controlledGlobalAttributes = [ 'experiment_id', 'project', 'frequency' ]
      self.globalAttributesInFn = [None,'@mip_id','model_id','experiment_id','@ensemble']
      self.requiredVarAttributes = ['long_name', 'units']
      self.drsMappings = {'variable':'@var', 'institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \
                        'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \
                        'frequency':'frequency',  'table':'@mip_id',
                        'project':'project_id'}

    elif project == 'ESA-CCI':
      lrdr = readVocab( 'esacci_vocabs/')
      self.varTables='FLAT'
      self.fNameSep = '-'
      self.checkVarType = False
      self.requiredGlobalAttributes = lrdr.getSimpleList( 'requiredGlobalAts.txt', bit=0 )
      self.controlledGlobalAttributes = ['platform','sensor','project','Conventions','institution','cdm_data_type','time_coverage_duration','spatial_resolution' ]
      self.controlledFnParts = ['level','cciProject','var','version']
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'#var','platform':'platform','sensor':'sensor','level':'#level', \
                'standard_name':'*standard_name', \
                'algorithm':'$algorithm:unset', 'frequency':'$frequency', \
                'spatial_resolution':'spatial_resolution', 'ecv':'@ecv','version':'#version','convention_version':'#gdsv'}
      self.globalAttributesInFn = [None,]
    elif project == '__dummy':
      self.requiredGlobalAttributes = map( lambda x: 'ga%s' % x, range(10) )
      self.controlledGlobalAttributes = [ ]
      self.globalAttributesInFn = [None,'ga2', 'ga3', 'ga4' ]
      self.requiredVarAttributes = ['long_name', 'standard_name', 'units']
      self.drsMappings = {'variable':'@var'}

####### used in checkStandardDims

    self.plevRequired = plevRequired
    self.plevValues = plevValues
    self.heightRequired = heightRequired
    self.heightValues = heightValues
    self.heightRange = heightRange

####### used in checkGrids
    self.rotatedPoleGrids = rotatedPoleGrids
    self.interpolatedGrids = interpolatedGrids
    self.doCheckGrids = self.projectV.id in ['CORDEX',]

####### used in checkFileName (freqIndex also used in checkByVar)

    if self.projectV.id == 'CORDEX':
      self.fnParts = NT_fnParts( len=[8,9], fxLen=[8,],  unfLen=[9,], checkTLen=True, ixDomain=1, ixFreq=7 )
    elif self.projectV.id == 'CMIP5':
      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None )
    elif self.projectV.id == 'SPECS':
      self.fnParts = lrdr.getEvalAssign( 'fnParts_sv0101.txt' )
    elif self.projectV.id == 'CCMI':
      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None )
    elif self.projectV.id == 'ESA-CCI':
      self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=1 )
      self.trangeType = 'ESA-CCI'
    elif self.projectV.id == '__dummy':
      self.fnParts = NT_fnParts( len=[4,5], fxLen=[4,],  unfLen=[5,], checkTLen=False, ixDomain=None, ixFreq=1 )

{'fraction_non_alphanumeric': 0.1896778868322245, 'fraction_numerical': 0.0363283051182399, 'mean_word_length': 4.188482570061518, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4470775', 'n_tokens_mistral': 11734, 'n_tokens_neox': 10611, 'n_words': 1951}