<commit_before>from os.path import join, abspath, dirname, getsize, curdir
from netCDF4 import Dataset
from flyingpigeon import config

DIR_SHP = config.shapefiles_dir()

import logging
logger = logging.getLogger(__name__)

def has_Lambert_Conformal(nc_files):
  if type(nc_files) != list:
      nc_files = [nc_files]
  for nc_file in nc_files:
    ds = Dataset(nc_file)
    if not 'Lambert_Conformal' in ds.variables.keys():
      return False
  return True

def call(resource=[], variable=None, dimension_map=None, calc=None,  
  calc_grouping= None, conform_units_to=None, memory_limit=None,  prefix=None, 
  regrid_destination=None, regrid_options='bil', level_range=None,
  geom=None, output_format_options=False, search_radius_mult=2., 
  select_nearest=False, select_ugid=None, spatial_wrapping=None, t_calendar=None, time_region=None, time_range=None,
  dir_output=curdir, output_format='nc'):
  '''
  ocgis operation call

  :param resource:
  :param variable: variable in the input file to be picked
  :param dimension_map: dimension map in case of unconventional starage of data 
  :param calc: ocgis calc syntax for calcultion opartion 
  :param calc_grouping: time aggregate grouping 
  :param conform_units_to: 
  :param memory_limit: limit the amout of data to be loaded into the memory at once if None(default) free memory is detected by birdhouse
  :param level_range: subset of given levels
  :param prefix: string for the file base name 
  :param regrid_destination: file path with netCDF file with grid for outout file

  :param geom: name of shapefile stored in birdhouse shape cabinet
  :param output_format_options: output options for netCDF e.g compression level()
  :param regrid_destination: file containing the targed grid (griddes.txt or netCDF file)
  :param regrid_options: methods for regridding: 
                          'bil' = Biliniar interpolation 
                          'bic' = Bicubic interpolation 
                          'dis' = Distance weighted average remapping
                          'nn' = nearest neighbour
                          'con' = First order concerative remapping
                          'laf' = largest area fraction reamapping
  :param search_radius_mult: search radius for point geometries. All included gridboxes will be returned
  :param select_nearest: neares neighbour selection for point geometries
  :param select_ugid: ugid for appropriate poligons 
  :param spatial_wrapping: how to handle coordinates in case of subsets, options: None(default), 'wrap', 'unwrap'
  :param time_region: select single month 
  :param time_range: sequence of two datetime.datetime objects to mark start and end point 
  :param dir_output (default= curdir):
  :param output_format:
  :return: output file path
  '''
  logger.info('Start ocgis module call function')
  from ocgis import OcgOperations, RequestDataset , env
  from ocgis.util.large_array import compute
  import uuid
  
  # prepare the environment 
  env.DIR_SHPCABINET = DIR_SHP
  env.OVERWRITE = True
  env.DIR_OUTPUT = dir_output
  
  if geom != None:
    spatial_reorder = True
    spatial_wrapping = 'wrap'
  else: 
    spatial_reorder = False
    spatial_wrapping = None
  
  
  if prefix == None:
    prefix = str(uuid.uuid1()) 
  env.PREFIX = prefix
  if output_format_options == False: 
    output_format_options = None
  elif output_format_options == True:
    output_format_options={'data_model': 'NETCDF4', # NETCDF4_CLASSIC
                         'variable_kwargs': {'zlib': True, 'complevel': 9}}
  else:
    logger.info('output_format_options are set to %s ' % ( output_format_options ))
  
  if type(resource) != list: 
    resource = list([resource])
  # execute ocgis 
  logger.info('Execute ocgis module call function')
  
  time_range = eval_timerange(resource, time_range)

  if has_Lambert_Conformal(resource) == True and not geom == None:
    logger.debug('input has Lambert_Conformal projection and can not subsetted with geom')
    output = None
  else:
    try:
      rd = RequestDataset(resource, variable=variable, level_range=level_range,
        dimension_map=dimension_map, conform_units_to=conform_units_to, 
        time_region=time_region,t_calendar=t_calendar, time_range=time_range)
      ops = OcgOperations(dataset=rd,
        output_format_options=output_format_options,
        spatial_wrapping=spatial_wrapping,
        spatial_reorder=spatial_reorder,
        # options=options,
        calc=calc,
        calc_grouping=calc_grouping,
        geom=geom,
        output_format=output_format,
        prefix=prefix,
        search_radius_mult=search_radius_mult,
        select_nearest=select_nearest,
        select_ugid=select_ugid, 
        add_auxiliary_files=False)
      logger.info('OcgOperations set')<commit_msg>time range validation only it time range is set<commit_after>from os.path import join, abspath, dirname, getsize, curdir
from netCDF4 import Dataset
from flyingpigeon import config

DIR_SHP = config.shapefiles_dir()

import logging
logger = logging.getLogger(__name__)

def has_Lambert_Conformal(nc_files):
  if type(nc_files) != list:
      nc_files = [nc_files]
  for nc_file in nc_files:
    ds = Dataset(nc_file)
    if not 'Lambert_Conformal' in ds.variables.keys():
      return False
  return True

def call(resource=[], variable=None, dimension_map=None, calc=None,  
  calc_grouping= None, conform_units_to=None, memory_limit=None,  prefix=None, 
  regrid_destination=None, regrid_options='bil', level_range=None,
  geom=None, output_format_options=False, search_radius_mult=2., 
  select_nearest=False, select_ugid=None, spatial_wrapping=None, t_calendar=None, time_region=None, time_range=None,
  dir_output=curdir, output_format='nc'):
  '''
  ocgis operation call

  :param resource:
  :param variable: variable in the input file to be picked
  :param dimension_map: dimension map in case of unconventional starage of data 
  :param calc: ocgis calc syntax for calcultion opartion 
  :param calc_grouping: time aggregate grouping 
  :param conform_units_to: 
  :param memory_limit: limit the amout of data to be loaded into the memory at once if None(default) free memory is detected by birdhouse
  :param level_range: subset of given levels
  :param prefix: string for the file base name 
  :param regrid_destination: file path with netCDF file with grid for outout file

  :param geom: name of shapefile stored in birdhouse shape cabinet
  :param output_format_options: output options for netCDF e.g compression level()
  :param regrid_destination: file containing the targed grid (griddes.txt or netCDF file)
  :param regrid_options: methods for regridding: 
                          'bil' = Biliniar interpolation 
                          'bic' = Bicubic interpolation 
                          'dis' = Distance weighted average remapping
                          'nn' = nearest neighbour
                          'con' = First order concerative remapping
                          'laf' = largest area fraction reamapping
  :param search_radius_mult: search radius for point geometries. All included gridboxes will be returned
  :param select_nearest: neares neighbour selection for point geometries
  :param select_ugid: ugid for appropriate poligons 
  :param spatial_wrapping: how to handle coordinates in case of subsets, options: None(default), 'wrap', 'unwrap'
  :param time_region: select single month 
  :param time_range: sequence of two datetime.datetime objects to mark start and end point 
  :param dir_output (default= curdir):
  :param output_format:
  :return: output file path
  '''
  logger.info('Start ocgis module call function')
  from ocgis import OcgOperations, RequestDataset , env
  from ocgis.util.large_array import compute
  import uuid
  
  # prepare the environment 
  env.DIR_SHPCABINET = DIR_SHP
  env.OVERWRITE = True
  env.DIR_OUTPUT = dir_output
  
  if geom != None:
    spatial_reorder = True
    spatial_wrapping = 'wrap'
  else: 
    spatial_reorder = False
    spatial_wrapping = None
  
  
  if prefix == None:
    prefix = str(uuid.uuid1()) 
  env.PREFIX = prefix
  if output_format_options == False: 
    output_format_options = None
  elif output_format_options == True:
    output_format_options={'data_model': 'NETCDF4', # NETCDF4_CLASSIC
                         'variable_kwargs': {'zlib': True, 'complevel': 9}}
  else:
    logger.info('output_format_options are set to %s ' % ( output_format_options ))
  
  if type(resource) != list: 
    resource = list([resource])
  # execute ocgis 
  logger.info('Execute ocgis module call function')
  
  if time_range != None:
    time_range = eval_timerange(resource, time_range)

  if has_Lambert_Conformal(resource) == True and not geom == None:
    logger.debug('input has Lambert_Conformal projection and can not subsetted with geom')
    output = None
  else:
    try:
      rd = RequestDataset(resource, variable=variable, level_range=level_range,
        dimension_map=dimension_map, conform_units_to=conform_units_to, 
        time_region=time_region,t_calendar=t_calendar, time_range=time_range)
      ops = OcgOperations(dataset=rd,
        output_format_options=output_format_options,
        spatial_wrapping=spatial_wrapping,
        spatial_reorder=spatial_reorder,
        # options=options,
        calc=calc,
        calc_grouping=calc_grouping,
        geom=geom,
        output_format=output_format,
        prefix=prefix,
        search_radius_mult=search_radius_mult,
        select_nearest=select_nearest,
        select_ugid=select_ugid, 
        add_auxiliary_files=False)
      logger.info('OcgOperations set')
{'fraction_non_alphanumeric': 0.07274047940230362, 'fraction_numerical': 0.001245200788627166, 'mean_word_length': 3.6582890285161915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2461193', 'n_tokens_mistral': 2942, 'n_tokens_neox': 2855, 'n_words': 840}