<commit_before>#!/usr/bin/env python
'''
@author <NAME> <<EMAIL>>
@file ion/services/dm/utility/granule_utils.py
@date Wed Jul 18 09:00:22 EDT 2012
@description Utilities for crafting granules into a coverage
'''

from pyon.core.exception import BadRequest
from pyon.ion.granule import TaxyTool, RecordDictionaryTool, build_granule 
from pyon.util.arg_check import validate_equal
from coverage_model.coverage import GridDomain, CRS, AxisTypeEnum, MutabilityEnum, GridShape, SimplexCoverage, RangeDictionary, ParameterContext
import dateutil.parser
import netCDF4
import time
import datetime
import numpy as np
'''
Assuming all values are np.float64 except data which is int8
'''
class CoverageCraft(object):
    tx = TaxyTool()
    tx.add_taxonomy_set('temp','long name for temp')
    tx.add_taxonomy_set('cond','long name for cond')
    tx.add_taxonomy_set('depth','')
    tx.add_taxonomy_set('lat','long name for latitude')
    tx.add_taxonomy_set('lon','long name for longitude')
    tx.add_taxonomy_set('time','long name for time')
    tx.add_taxonomy_set('data', 'arbitrary data uint8')

    def __init__(self,coverage=None, granule=None):
        self.rdt = RecordDictionaryTool(self.tx)
        if coverage is None:
            self.coverage = self.create_coverage()
        else:
            self.coverage = coverage
        
        if granule is not None:
            self.from_granule(granule)


    def from_granule(self, granule):
        tt = TaxyTool.load_from_granule(granule)
        validate_equal(tt,self.tx, "The taxonomies don't match up.")
        rdt = RecordDictionaryTool.load_from_granule(granule)
        self.rdt = rdt
        return rdt

    def add_granule(self, granule=None):
        if granule is not None:
            self.from_granule(granule)
        start_index = self.coverage.num_timesteps 
        elements = self.rdt._shp[0]
        if self.coverage.num_timesteps == 1:
            start_index = 0
            self.coverage.insert_timesteps(elements - 1)
        else:
            self.coverage.insert_timesteps(elements)

        for k,v in self.rdt.iteritems():
            print "key: %s" % k
            print "value: %s" % v
            slice_ = slice(start_index,None)
            print "slice: %s" % slice_
            self.coverage.set_parameter_values(k,tdoa=slice_, value=v)


    def to_granule(self):
        return build_granule('coverage_craft', self.tx, self.rdt)

    @classmethod
    def create_coverage(cls):
        rdict = RangeDictionary()
        rdict.items = {
            'coords' : ['time', 'lat', 'lon','depth'],
            'vars'   : ['temp', 'cond', 'data']
        }

        tcrs = CRS([AxisTypeEnum.TIME])
        scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])

        tdom = GridDomain(GridShape('temporal'), tcrs, MutabilityEnum.EXTENSIBLE)
        sdom = GridDomain(GridShape('spatial', [1]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now
    
        scov = SimplexCoverage('sample grid coverage_model', rdict, sdom, tdom)

        for val in cls.tx._by_nick_names.iterkeys():
            cls.create_parameters(scov,rdict,val)

        return scov

    @classmethod
    def create_parameters(cls, coverage,rdict, parameter):
        if parameter is not 'data':
            pcontext = ParameterContext(parameter, param_type=np.float64)
        else:
            pcontext = ParameterContext(parameter, param_type=np.int8)
        #--------------------------------------------------------------------------------
        # Coordinate parameters
        #--------------------------------------------------------------------------------
        if parameter is "time":
            pcontext.uom = 'urn:ogc:def:uom:UCUM::s'
            pcontext.description = 'time'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.TIME
        elif parameter is "lat":
            pcontext.uom = "degrees_north"
            pcontext.description = 'wgs84 latitude'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.LAT
        elif parameter is "lon":
            pcontext.uom = "degrees_east"
            pcontext.description = 'wgs84 longitude'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.LON
        elif parameter is "depth":
            pcontext.uom = 'urn:ogc:def:uom:UCUM::m'
            pcontext.description = 'depth in metres'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.HEIGHT
        #--------------------------------------------------------------------------------
        # Value parameters
        #--------------------------------------------------------------------------------
        elif parameter is "temp":
            pcontext.uom = 'urn:ogc:def:uom:UCUM::c'
            pcontext.description = 'degrees centigrade'
            pcontext.fill_value = np.float64(0e0)
        elif parameter is "cond":
            pcontext.uom = "???"
            pcontext.description = 'conductivity'
            pcontext.fill_value = np.float64(0e0)
        elif parameter is "data":
            pcontext.uom = "byte"
            pcontext.description = 'arbitrary data'
            pcontext.fill_value = np.int8(0x0)
        else:
            raise BadRequest('The parameter is not explicitly defined and therefore not accepted')

        coverage.append_parameter(pcontext)

    @staticmethod
    def ts_to_units(units, val):
        '''
        Converts a unix timestamp into various formats
        Example:
        ts = time.time()
        CoverageCraft.ts_to_units('days since 2000-01-01', ts)
        '''
        if 'iso' in units:
            return time.strftime('%Y-%d-%mT%H:%M:%S', time.gmtime(val))
        elif 'since' in units:
            t = netCDF4.netcdftime.utime(units)
            return t.date2num(datetime.datetime.fromtimestamp(val))
        else:
            return val




         
    @staticmethod
    def units_to_ts(units, val):
        '''
        Converts known time formats into a unix timestamp
        Example:
        ts = CoverageCraft.units_to_ts('days since 2000-01-01', 1200)
        '''
        if 'since' in units:
            t = netCDF4.netcdftime.utime(units)
            dtg = t.num2date(val)
            return time.mktime(dtg.timetuple())
        elif 'iso' in units:
            t = dateutil.parser.parse(val)
            return time.mktime(t.timetuple())
        else:
            return val
        


    def build_coverage(self):
        pass

<commit_msg>Luke: added relative time index utility<commit_after>#!/usr/bin/env python
'''
@author <NAME> <<EMAIL>>
@file ion/services/dm/utility/granule_utils.py
@date Wed Jul 18 09:00:22 EDT 2012
@description Utilities for crafting granules into a coverage
'''

from pyon.core.exception import BadRequest
from pyon.ion.granule import TaxyTool, RecordDictionaryTool, build_granule 
from pyon.util.arg_check import validate_equal
from coverage_model.coverage import GridDomain, CRS, AxisTypeEnum, MutabilityEnum, GridShape, SimplexCoverage, RangeDictionary, ParameterContext
import dateutil.parser
import netCDF4
import time
import datetime
import numpy as np
'''
Assuming all values are np.float64 except data which is int8
'''
class CoverageCraft(object):
    tx = TaxyTool()
    tx.add_taxonomy_set('temp','long name for temp')
    tx.add_taxonomy_set('cond','long name for cond')
    tx.add_taxonomy_set('depth','')
    tx.add_taxonomy_set('lat','long name for latitude')
    tx.add_taxonomy_set('lon','long name for longitude')
    tx.add_taxonomy_set('time','long name for time')
    tx.add_taxonomy_set('data', 'arbitrary data uint8')

    def __init__(self,coverage=None, granule=None):
        self.rdt = RecordDictionaryTool(self.tx)
        if coverage is None:
            self.coverage = self.create_coverage()
        else:
            self.coverage = coverage
        
        if granule is not None:
            self.from_granule(granule)


    def from_granule(self, granule):
        tt = TaxyTool.load_from_granule(granule)
        validate_equal(tt,self.tx, "The taxonomies don't match up.")
        rdt = RecordDictionaryTool.load_from_granule(granule)
        self.rdt = rdt
        return rdt

    def add_granule(self, granule=None):
        if granule is not None:
            self.from_granule(granule)
        start_index = self.coverage.num_timesteps 
        elements = self.rdt._shp[0]
        if self.coverage.num_timesteps == 1:
            start_index = 0
            self.coverage.insert_timesteps(elements - 1)
        else:
            self.coverage.insert_timesteps(elements)

        for k,v in self.rdt.iteritems():
            print "key: %s" % k
            print "value: %s" % v
            slice_ = slice(start_index,None)
            print "slice: %s" % slice_
            self.coverage.set_parameter_values(k,tdoa=slice_, value=v)


    def to_granule(self):
        return build_granule('coverage_craft', self.tx, self.rdt)

    @classmethod
    def create_coverage(cls):
        rdict = RangeDictionary()
        rdict.items = {
            'coords' : ['time', 'lat', 'lon','depth'],
            'vars'   : ['temp', 'cond', 'data']
        }

        tcrs = CRS([AxisTypeEnum.TIME])
        scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])

        tdom = GridDomain(GridShape('temporal'), tcrs, MutabilityEnum.EXTENSIBLE)
        sdom = GridDomain(GridShape('spatial', [1]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now
    
        scov = SimplexCoverage('sample grid coverage_model', rdict, sdom, tdom)

        for val in cls.tx._by_nick_names.iterkeys():
            cls.create_parameters(scov,rdict,val)

        return scov

    @classmethod
    def create_parameters(cls, coverage,rdict, parameter):
        if parameter is not 'data':
            pcontext = ParameterContext(parameter, param_type=np.float64)
        else:
            pcontext = ParameterContext(parameter, param_type=np.int8)
        #--------------------------------------------------------------------------------
        # Coordinate parameters
        #--------------------------------------------------------------------------------
        if parameter is "time":
            pcontext.uom = 'seconds since 1970-01-01'
            pcontext.description = 'time'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.TIME
        elif parameter is "lat":
            pcontext.uom = "degrees_north"
            pcontext.description = 'wgs84 latitude'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.LAT
        elif parameter is "lon":
            pcontext.uom = "degrees_east"
            pcontext.description = 'wgs84 longitude'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.LON
        elif parameter is "depth":
            pcontext.uom = 'urn:ogc:def:uom:UCUM::m'
            pcontext.description = 'depth in metres'
            pcontext.fill_value = np.float64(0e0)
            pcontext.axis = AxisTypeEnum.HEIGHT
        #--------------------------------------------------------------------------------
        # Value parameters
        #--------------------------------------------------------------------------------
        elif parameter is "temp":
            pcontext.uom = 'urn:ogc:def:uom:UCUM::c'
            pcontext.description = 'degrees centigrade'
            pcontext.fill_value = np.float64(0e0)
        elif parameter is "cond":
            pcontext.uom = "???"
            pcontext.description = 'conductivity'
            pcontext.fill_value = np.float64(0e0)
        elif parameter is "data":
            pcontext.uom = "byte"
            pcontext.description = 'arbitrary data'
            pcontext.fill_value = np.int8(0x0)
        else:
            raise BadRequest('The parameter is not explicitly defined and therefore not accepted')

        coverage.append_parameter(pcontext)
        
    @classmethod
    def get_relative_time(cls, coverage, time):
        '''
        Determines the relative time in the coverage model based on a given time
        The time must match the coverage's time units
        '''
        pc = coverage.get_parameter_context('time')
        units = pc.uom
        if 'iso' in units:
            return None # Not sure how to implement this....  How do you compare iso strings effectively?
        values = coverage.get_parameter_values('time')
        return cls.binary_search(values,time)
        

    @classmethod
    def binary_search(cls, array, value, imin=None, imax=None):
        '''
        Binary search against an array, identifies the best index
        Best index is one that meets the following:
          If the value lies between two elements in the array it picks the first element
          If the value is greater than the last value in the array it chooses the last element
          If the value is less than the first value in the array it chooses the first element
        '''
        if imin is None:
            imin = 0
        if imax is None:
            imax = len(array) - 1
        if imin == imax or imax == imin+1:
            return imin
        mid = (imax - imin) / 2 + imin

        # Short circuit entire section
        if array[imin] > value:
            return imin

        if array[imax] < value:
            return imax


        if array[mid] == value:
            return mid
        elif array[mid] < value:
            return cls.binary_search(array,value,mid,imax)
        elif array[mid] > value:
            return cls.binary_search(array, value, imin, mid)
        else:
            return mid


            




    @staticmethod
    def ts_to_units(units, val):
        '''
        Converts a unix timestamp into various formats
        Example:
        ts = time.time()
        CoverageCraft.ts_to_units('days since 2000-01-01', ts)
        '''
        if 'iso' in units:
            return time.strftime('%Y-%d-%mT%H:%M:%S', time.gmtime(val))
        elif 'since' in units:
            t = netCDF4.netcdftime.utime(units)
            return t.date2num(datetime.datetime.fromtimestamp(val))
        else:
            return val


    @staticmethod
    def units_to_ts(units, val):
        '''
        Converts known time formats into a unix timestamp
        Example:
        ts = CoverageCraft.units_to_ts('days since 2000-01-01', 1200)
        '''
        if 'since' in units:
            t = netCDF4.netcdftime.utime(units)
            dtg = t.num2date(val)
            return time.mktime(dtg.timetuple())
        elif 'iso' in units:
            t = dateutil.parser.parse(val)
            return time.mktime(t.timetuple())
        else:
            return val
        


    def build_coverage(self):
        pass


{'fraction_non_alphanumeric': 0.13645812378461744, 'fraction_numerical': 0.011533561322336216, 'mean_word_length': 1.949179355349021, 'pattern_counts': {'":': 14, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386664', 'n_tokens_mistral': 4503, 'n_tokens_neox': 4281, 'n_words': 1046}