<commit_before># -*- coding: utf-8 -*-
# Author: <NAME> <<EMAIL>>

# pylint: disable=C0103,R0903

from __future__ import absolute_import

import json

from itertools import chain
from urllib2 import urlopen
from urllib import urlencode
from datetime import datetime
from functools import partial
from sunpy.net import attr
from sunpy.net.hek import attrs
from sunpy.net.vso import attrs as v_attrs
from sunpy.util.util import unique
from sunpy.util.xml import xml_to_dict

DEFAULT_URL = 'http://www.lmsal.com/hek/her'

def _freeze(obj):
    """ Create hashable representation of result dict. """
    if isinstance(obj, dict):
        return tuple((k, _freeze(v)) for k, v in obj.iteritems())
    if isinstance(obj, list):
        return tuple(_freeze(elem) for elem in obj)
    return obj


class HEKClient(object):
    # FIXME: Types!
    
    default = {
        'cosec': '2',
        'cmd': 'search',
        'type': 'column',
        'event_type': '**',
    }
    # Default to full disk.
    attrs.walker.apply(attrs.SpatialRegion(), {}, default)
    
    def __init__(self, url=DEFAULT_URL):
        self.url = url
    
    def _download(self, data):
        page = 1        
        results = []
        
        while True:
            data['page'] = page
            result = json.load(urlopen(self.url, urlencode(data)))
            results.extend(result['result'])
            
            if not result['overmax']:
                return map(Response, results)
            page += 1
    
    def query(self, *query):
        query = attr.and_(*query)
        
        data = attrs.walker.create(query, {})
        ndata = []
        for elem in data:
            new = self.default.copy()
            new.update(elem)
            ndata.append(new)
        
        if len(ndata) == 1:
            return self._download(ndata[0])
        else:
            return self.merge(self._download(data) for data in ndata)
    
    def merge(self, responses):
        return list(unique(chain.from_iterable(responses), _freeze))


class Response(dict):
    @property
    def vso_time(self):
        return v_attrs.Time(
            datetime.strptime(self['event_starttime'], "%Y-%m-%dT%H:%M:%S"),
            datetime.strptime(self['event_endtime'], "%Y-%m-%dT%H:%M:%S")
        )
    
    @property
    def vso_instrument(self):
        if self['obs_instrument'] == 'HEK':
            raise ValueError("No instrument contained.")
        return v_attrs.Instrument(self['obs_instrument'])
    
    @property
    def vso_all(self):
        return attr.and_(self.vso_time, self.vso_instrument)
    
    def get_voevent(self, as_dict=True):
        """Retrieves the VOEvent object associated with a given event and
        returns it as either a Python dictionary or an XML string."""
        
        base_url = "http://www.lmsal.com/hek/her?"
        
        # Build URL
        params = {                                                      
            "cmd": "export-voevent",
            "cosec": 1,
            "ivorn": self['kb_archivid']
        }
        url = base_url + urlencode(params)
        
        # Query and read response
        response = urlopen(url).read()<commit_msg>Added option to override base_url when querying HEK for VOEvent data<commit_after># -*- coding: utf-8 -*-
# Author: <NAME> <<EMAIL>>

# pylint: disable=C0103,R0903

from __future__ import absolute_import

import json

from itertools import chain
from urllib2 import urlopen
from urllib import urlencode
from datetime import datetime
from functools import partial
from sunpy.net import attr
from sunpy.net.hek import attrs
from sunpy.net.vso import attrs as v_attrs
from sunpy.util.util import unique
from sunpy.util.xml import xml_to_dict

DEFAULT_URL = 'http://www.lmsal.com/hek/her'

def _freeze(obj):
    """ Create hashable representation of result dict. """
    if isinstance(obj, dict):
        return tuple((k, _freeze(v)) for k, v in obj.iteritems())
    if isinstance(obj, list):
        return tuple(_freeze(elem) for elem in obj)
    return obj


class HEKClient(object):
    # FIXME: Types!
    
    default = {
        'cosec': '2',
        'cmd': 'search',
        'type': 'column',
        'event_type': '**',
    }
    # Default to full disk.
    attrs.walker.apply(attrs.SpatialRegion(), {}, default)
    
    def __init__(self, url=DEFAULT_URL):
        self.url = url
    
    def _download(self, data):
        page = 1        
        results = []
        
        while True:
            data['page'] = page
            result = json.load(urlopen(self.url, urlencode(data)))
            results.extend(result['result'])
            
            if not result['overmax']:
                return map(Response, results)
            page += 1
    
    def query(self, *query):
        query = attr.and_(*query)
        
        data = attrs.walker.create(query, {})
        ndata = []
        for elem in data:
            new = self.default.copy()
            new.update(elem)
            ndata.append(new)
        
        if len(ndata) == 1:
            return self._download(ndata[0])
        else:
            return self.merge(self._download(data) for data in ndata)
    
    def merge(self, responses):
        return list(unique(chain.from_iterable(responses), _freeze))


class Response(dict):
    @property
    def vso_time(self):
        return v_attrs.Time(
            datetime.strptime(self['event_starttime'], "%Y-%m-%dT%H:%M:%S"),
            datetime.strptime(self['event_endtime'], "%Y-%m-%dT%H:%M:%S")
        )
    
    @property
    def vso_instrument(self):
        if self['obs_instrument'] == 'HEK':
            raise ValueError("No instrument contained.")
        return v_attrs.Instrument(self['obs_instrument'])
    
    @property
    def vso_all(self):
        return attr.and_(self.vso_time, self.vso_instrument)
    
    def get_voevent(self, as_dict=True, 
                    base_url="http://www.lmsal.com/hek/her?"):
        """Retrieves the VOEvent object associated with a given event and
        returns it as either a Python dictionary or an XML string."""

        # Build URL
        params = {                                                      
            "cmd": "export-voevent",
            "cosec": 1,
            "ivorn": self['kb_archivid']
        }
        url = base_url + urlencode(params)
        
        # Query and read response
        response = urlopen(url).read()
{'fraction_non_alphanumeric': 0.11699813548788067, 'fraction_numerical': 0.004972032318210068, 'mean_word_length': 2.043498817966903, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2852415', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1934, 'n_words': 488}