<commit_before>from pyon.core.exception import BadRequest
from pyon.ion.process import StandaloneProcess
from pyon.util.file_sys import FileSystem
from pyon.util.log import log

from ion.services.dm.inventory.dataset_management_service import DatasetManagementService

from coverage_model import AbstractCoverage
#from coverage_model.parameter_types import QuantityType

from xml.dom.minidom import parse, parseString
from zipfile import ZipFile

import base64
import os
import urllib
import xml.dom.minidom
import StringIO

class RegistrationProcess(StandaloneProcess):

    def on_start(self):
        #these values should come in from a config file, maybe pyon.yml
        self.pydap_host = self.CFG.get_safe('server.pydap.host', 'localhost')
        self.pydap_port = self.CFG.get_safe('server.pydap.port', '8001')
        self.pydap_url  = 'http://%s:%s/' % (self.pydap_host, self.pydap_port)
        self.pydap_data_path = self.CFG.get_safe('server.pydap.data_path', 'RESOURCE:ext/pydap')
        self.datasets_xml_path = self.CFG.get_safe('server.pydap.datasets_xml_path', "RESOURCE:ext/datasets.xml")
        self.pydap_data_path = FileSystem.get_extended_url(self.pydap_data_path) + '/'

        filename = self.datasets_xml_path.split('/')[-1]
        base = '/'.join(self.datasets_xml_path.split('/')[:-1])
        real_path = FileSystem.get_extended_url(base)
        self.datasets_xml_path = os.path.join(real_path, filename)
        self.setup_filesystem(real_path)

    def setup_filesystem(self, path):
        if os.path.exists(os.path.join(path,'datasets.xml')):
            return
        with open(os.path.join(path,'datasets.xml'),'w') as f:
            f.write(datasets_xml)

    def register_dap_dataset(self, dataset_id, data_product_name=''):
        coverage_path = DatasetManagementService._get_coverage_path(dataset_id)
        try:
            self.add_dataset_to_xml(coverage_path=coverage_path, product_name=data_product_name)
            self.create_symlink(coverage_path, self.pydap_data_path)
        except: # We don't re-raise to prevent clients from bombing out...
            log.exception('Problem registering dataset')
            log.error('Failed to register dataset for coverage path %s' % coverage_path)
    
    def create_symlink(self, coverage_path, pydap_path):
        paths = os.path.split(coverage_path)
        os.symlink(coverage_path, pydap_path + paths[1])

    def add_dataset_to_xml(self, coverage_path, product_name=''):
        dom1 = parse(self.datasets_xml_path)
        xml_str = self.get_dataset_xml(coverage_path, product_name)
        dom2 = parseString(xml_str)

        erddap_datasets_element = dom1.getElementsByTagName('erddapDatasets')[0]
        erddap_datasets_element.appendChild(dom2.getElementsByTagName('dataset')[0])

        with open(self.datasets_xml_path, 'w') as f:
            dom1.writexml(f)
    
    def get_errdap_name_map(self, names):
        result = {}
        for name in names:
            if name == 'lon':
                result[name] = 'longitude'
            elif name == 'lat':
                result[name] = 'latitude'
            else:
                result[name] = name
        return result

    def get_dataset_xml(self, coverage_path, product_name=''):
        #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html
        result = ''
        paths = os.path.split(coverage_path)
        cov = AbstractCoverage.load(coverage_path)
        doc = xml.dom.minidom.Document()
        
        #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} 
        
        #Get lists of variables with unique sets of dimensions.
        #Datasets can only have variables with the same sets of dimensions

        if not cov.list_parameters():
            raise BadRequest('Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' %(coverage_path, cov))

        datasets = {}
        for key in cov.list_parameters():
            pc = cov.get_parameter_context(key)
            
            #if not isinstance(pc.param_type, QuantityType):
            #    continue

            param = cov.get_parameter(key)
            dims = (cov.temporal_parameter_name,)
            if len(param.shape) == 2:
                dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name)

            if not dims in datasets.keys():
                datasets[dims] = []
            

            datasets[dims].append(key)
        

        index = 0
        if not datasets:
            raise BadRequest('Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' %( coverage_path, cov))
        
        for dims, vars in datasets.iteritems():
            erd_name_map = self.get_errdap_name_map(vars) 
            
            if len(vars)==1:
                raise BadRequest('A dataset needs a proper range, not just the temporal dimension. %s\n%s' %( coverage_path, cov))

            if not (len(dims) == 1 and dims[0] == vars[0]):
                dataset_element = doc.createElement('dataset')
                dataset_element.setAttribute('type', 'EDDGridFromDap')
                dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index))
                dataset_element.setAttribute('active', 'True')

                source_element = doc.createElement('sourceUrl')
                text_node = doc.createTextNode(self.pydap_url + paths[1])
                source_element.appendChild(text_node)
                dataset_element.appendChild(source_element)

                reload_element = doc.createElement('reloadEveryNMinutes')
                text_node = doc.createTextNode('5')
                reload_element.appendChild(text_node)
                dataset_element.appendChild(reload_element)
                

                add_attributes_element = doc.createElement('addAttributes')

                atts = {}
                atts['title'] = product_name or urllib.unquote(cov.name)
                atts['infoUrl'] = self.pydap_url + paths[1]
                atts['institution'] = 'OOI'
                atts['Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"
                atts['license'] = '[standard]'
                atts['summary'] = cov.name
                atts['cdm_data_type'] = 'Grid'
                atts['standard_name_vocabulary'] = 'CF-12'
                
                for key, val in atts.iteritems():
                    att_element = doc.createElement('att')
                    att_element.setAttribute('name', key)
                    text_node = doc.createTextNode(val)
                    att_element.appendChild(text_node)
                    add_attributes_element.appendChild(att_element)

                if len(add_attributes_element.childNodes) > 0:
                    dataset_element.appendChild(add_attributes_element)

                for var_name in vars:
                    var = cov.get_parameter_context(var_name)
                    
                    units = "unknown"
                    if hasattr(var,'uom') and var.uom:<commit_msg>ERDDAP OOI Release Support<commit_after>from pyon.core.exception import BadRequest
from pyon.ion.process import StandaloneProcess
from pyon.util.file_sys import FileSystem
from pyon.util.log import log

from ion.services.dm.inventory.dataset_management_service import DatasetManagementService

from coverage_model import AbstractCoverage
#from coverage_model.parameter_types import QuantityType

from xml.dom.minidom import parse, parseString
from zipfile import ZipFile

import base64
import os
import urllib
import xml.dom.minidom
import StringIO

class RegistrationProcess(StandaloneProcess):

    def on_start(self):
        #these values should come in from a config file, maybe pyon.yml
        self.pydap_host = self.CFG.get_safe('server.pydap.host', 'localhost')
        self.pydap_port = self.CFG.get_safe('server.pydap.port', '8001')
        self.pydap_url  = 'http://%s:%s/' % (self.pydap_host, self.pydap_port)
        self.pydap_data_path = self.CFG.get_safe('server.pydap.data_path', 'RESOURCE:ext/pydap')
        self.datasets_xml_path = self.CFG.get_safe('server.pydap.datasets_xml_path', "RESOURCE:ext/datasets.xml")
        self.pydap_data_path = FileSystem.get_extended_url(self.pydap_data_path) + '/'

        filename = self.datasets_xml_path.split('/')[-1]
        base = '/'.join(self.datasets_xml_path.split('/')[:-1])
        real_path = FileSystem.get_extended_url(base)
        self.datasets_xml_path = os.path.join(real_path, filename)
        self.setup_filesystem(real_path)

    def setup_filesystem(self, path):
        if os.path.exists(os.path.join(path,'datasets.xml')):
            return
        with open(os.path.join(path,'datasets.xml'),'w') as f:
            f.write(datasets_xml)

    def register_dap_dataset(self, dataset_id, data_product_name=''):
        coverage_path = DatasetManagementService._get_coverage_path(dataset_id)
        try:
            self.add_dataset_to_xml(coverage_path=coverage_path, product_name=data_product_name)
            self.create_symlink(coverage_path, self.pydap_data_path)
        except: # We don't re-raise to prevent clients from bombing out...
            log.exception('Problem registering dataset')
            log.error('Failed to register dataset for coverage path %s' % coverage_path)
    
    def create_symlink(self, coverage_path, pydap_path):
        paths = os.path.split(coverage_path)
        os.symlink(coverage_path, pydap_path + paths[1])

    def add_dataset_to_xml(self, coverage_path, product_name=''):
        dom1 = parse(self.datasets_xml_path)
        xml_str = self.get_dataset_xml(coverage_path, product_name)
        dom2 = parseString(xml_str)

        erddap_datasets_element = dom1.getElementsByTagName('erddapDatasets')[0]
        erddap_datasets_element.appendChild(dom2.getElementsByTagName('dataset')[0])

        with open(self.datasets_xml_path, 'w') as f:
            dom1.writexml(f)
    
    def get_errdap_name_map(self, names):
        result = {}
        for name in names:
            if name == 'lon':
                result[name] = 'longitude'
            elif name == 'lat':
                result[name] = 'latitude'
            else:
                result[name] = name
        return result

    def get_dataset_xml(self, coverage_path, product_name=''):
        #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html
        result = ''
        paths = os.path.split(coverage_path)
        cov = AbstractCoverage.load(coverage_path)
        doc = xml.dom.minidom.Document()
        
        #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} 
        
        #Get lists of variables with unique sets of dimensions.
        #Datasets can only have variables with the same sets of dimensions

        if not cov.list_parameters():
            raise BadRequest('Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' %(coverage_path, cov))

        datasets = {}
        for key in cov.list_parameters():
            pc = cov.get_parameter_context(key)
            
            #if not isinstance(pc.param_type, QuantityType):
            #    continue

            param = cov.get_parameter(key)
            dims = (cov.temporal_parameter_name,)
            if len(param.shape) == 2:
                dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name)

            if not dims in datasets.keys():
                datasets[dims] = []
            

            datasets[dims].append(key)
        

        index = 0
        if not datasets:
            raise BadRequest('Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' %( coverage_path, cov))
        
        for dims, vars in datasets.iteritems():
            erd_name_map = self.get_errdap_name_map(vars) 
            
            if len(vars)==1:
                raise BadRequest('A dataset needs a proper range, not just the temporal dimension. %s\n%s' %( coverage_path, cov))

            if not (len(dims) == 1 and dims[0] == vars[0]):
                dataset_element = doc.createElement('dataset')
                dataset_element.setAttribute('type', 'EDDGridFromDap')
                dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index))
                dataset_element.setAttribute('active', 'True')

                source_element = doc.createElement('sourceUrl')
                text_node = doc.createTextNode(self.pydap_url + paths[1])
                source_element.appendChild(text_node)
                dataset_element.appendChild(source_element)

                reload_element = doc.createElement('reloadEveryNMinutes')
                text_node = doc.createTextNode('1440')
                reload_element.appendChild(text_node)
                dataset_element.appendChild(reload_element)

                update_element = doc.createElement('updateEveryNMillis')
                text_node = doc.createTextNode('1000')
                update_element.appendChild(text_node)
                dataset_element.appendChild(update_element)
                

                add_attributes_element = doc.createElement('addAttributes')

                atts = {}
                atts['title'] = product_name or urllib.unquote(cov.name)
                atts['infoUrl'] = self.pydap_url + paths[1]
                atts['institution'] = 'OOI'
                atts['Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"
                atts['license'] = '[standard]'
                atts['summary'] = cov.name
                atts['cdm_data_type'] = 'Grid'
                atts['standard_name_vocabulary'] = 'CF-12'
                
                for key, val in atts.iteritems():
                    att_element = doc.createElement('att')
                    att_element.setAttribute('name', key)
                    text_node = doc.createTextNode(val)
                    att_element.appendChild(text_node)
                    add_attributes_element.appendChild(att_element)

                if len(add_attributes_element.childNodes) > 0:
                    dataset_element.appendChild(add_attributes_element)

                for var_name in vars:
                    var = cov.get_parameter_context(var_name)
                    
                    units = "unknown"
                    if hasattr(var,'uom') and var.uom:
{'fraction_non_alphanumeric': 0.10622986772668083, 'fraction_numerical': 0.00527722568706737, 'mean_word_length': 2.358342922899885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 42}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2365206', 'n_tokens_mistral': 4388, 'n_tokens_neox': 4214, 'n_words': 816}