<commit_before>from __future__ import print_function, division, absolute_import #, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open
from future.moves.urllib.parse import urlparse, urlunparse, urlencode
from future.moves.urllib.request import urlopen
from future.moves.urllib.error import HTTPError

import os.path
from lxml import etree, objectify
import json
from astropy import time
import numpy as np
import logging
logger = logging.getLogger(__name__)

_install_dir = os.path.abspath(os.path.dirname(__file__))
_xsd_dir = os.path.join(_install_dir, 'xsd')
_sdmbuilder_xsd = os.path.join(_xsd_dir, 'SdmBuilderMessage.xsd')
_sdmbuilder_parser = objectify.makeparser(
        schema=etree.XMLSchema(file=_sdmbuilder_xsd))
_antflagger_xsd = os.path.join(_xsd_dir, 'AntFlaggerMessage.xsd')
_antflagger_parser = objectify.makeparser(
        schema=etree.XMLSchema(file=_antflagger_xsd))

_host = 'mctest.evla.nrao.edu'
_sdmpath = 'sdm-builder/offline'
_antpath = 'evla-mcaf-test/dataset'


class SDMBuilder(object):
    """ Use mcaf to create new SDM from bdf
    """

    _E = objectify.ElementMaker(annotate=False)

    def __init__(self, datasetId=None, outputDatasetId=None,
                 uid=None, dataSize=None,
                 numIntegrations=None, startTime=None, endTime=None,
                 calScanTime=None, annotation={},
                 host=_host, path=_sdmpath):
        self.datasetId = datasetId
        if outputDatasetId is None:
            self.outputDatasetId = 'realfast_{0}_{1}'.format(datasetId,
                                                             uid.rsplit('/')[-1])
        else:
            self.outputDatasetId = outputDatasetId
        self.uid = uid
        self.dataSize = dataSize
        self.numIntegrations = numIntegrations
        self.startTime = startTime
        self.endTime = endTime
        self.calScanTime = calScanTime
        self.annotation = annotation
        self.host = host
        self.path = path

    @property
    def _root(self):
        return self._E.SdmBuilderMessage(
                self._E.datasetId(self.datasetId),
                self._E.outputDatasetId(self.outputDatasetId),
                self._E.bdf(
                    self._E.uid(self.uid),
                    self._E.dataSize(self.dataSize),
                    self._E.numIntegrations(self.numIntegrations),
                    self._E.startTime(repr(self.startTime)),
                    self._E.endTime(repr(self.endTime)),
                    self._E.calScanTime(repr(self.calScanTime)),
                    ),
                self._E.annotation(
                    self._E.sValue(json.dumps(self.annotation)),
                    # annotation attributes:
                    {'issue': 'REALFAST_METADATA',
                        'details': 'Realfast candidate metadata'}
                    ),
                # SdmBuilderMessage attributes:
                {'timestamp': '%.12f' % time.Time.now().mjd,
                    'sender': 'realfast'}
                )

    @property
    def xml(self):
        return etree.tostring(self._root, xml_declaration=True,
                              pretty_print=False, standalone=True)

    @property
    def _url(self):
        query = urlencode({'xml': self.xml})
        url = urlunparse(('https', self.host, self.path, '', query, ''))
        return url

    def send(self):
        response_xml = urlopen(self._url, timeout=10).read()
        self.response = None
        if b'error' in response_xml:
            logger.warn("error in sdmbuilder response xml: {0}"
                        .format(response_xml))
            self.response = response_xml
        else:
            self.response = objectify.fromstring(response_xml,
                                                 parser=_sdmbuilder_parser)

    @property
    def location(self):
        try:
            return str(self.response.result.location)
        except AttributeError:
            logger.warn("No SDM generated.")
            return None


def makesdm(startTime, endTime, datasetId, data, calScanTime=None,<commit_msg>urlopen read timeout changed from 10 to 60 s<commit_after>from __future__ import print_function, division, absolute_import #, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open
from future.moves.urllib.parse import urlparse, urlunparse, urlencode
from future.moves.urllib.request import urlopen
from future.moves.urllib.error import HTTPError

import os.path
from lxml import etree, objectify
import json
from astropy import time
import numpy as np
import logging
logger = logging.getLogger(__name__)

_install_dir = os.path.abspath(os.path.dirname(__file__))
_xsd_dir = os.path.join(_install_dir, 'xsd')
_sdmbuilder_xsd = os.path.join(_xsd_dir, 'SdmBuilderMessage.xsd')
_sdmbuilder_parser = objectify.makeparser(
        schema=etree.XMLSchema(file=_sdmbuilder_xsd))
_antflagger_xsd = os.path.join(_xsd_dir, 'AntFlaggerMessage.xsd')
_antflagger_parser = objectify.makeparser(
        schema=etree.XMLSchema(file=_antflagger_xsd))

_host = 'mctest.evla.nrao.edu'
_sdmpath = 'sdm-builder/offline'
_antpath = 'evla-mcaf-test/dataset'


class SDMBuilder(object):
    """ Use mcaf to create new SDM from bdf
    """

    _E = objectify.ElementMaker(annotate=False)

    def __init__(self, datasetId=None, outputDatasetId=None,
                 uid=None, dataSize=None,
                 numIntegrations=None, startTime=None, endTime=None,
                 calScanTime=None, annotation={},
                 host=_host, path=_sdmpath):
        self.datasetId = datasetId
        if outputDatasetId is None:
            self.outputDatasetId = 'realfast_{0}_{1}'.format(datasetId,
                                                             uid.rsplit('/')[-1])
        else:
            self.outputDatasetId = outputDatasetId
        self.uid = uid
        self.dataSize = dataSize
        self.numIntegrations = numIntegrations
        self.startTime = startTime
        self.endTime = endTime
        self.calScanTime = calScanTime
        self.annotation = annotation
        self.host = host
        self.path = path

    @property
    def _root(self):
        return self._E.SdmBuilderMessage(
                self._E.datasetId(self.datasetId),
                self._E.outputDatasetId(self.outputDatasetId),
                self._E.bdf(
                    self._E.uid(self.uid),
                    self._E.dataSize(self.dataSize),
                    self._E.numIntegrations(self.numIntegrations),
                    self._E.startTime(repr(self.startTime)),
                    self._E.endTime(repr(self.endTime)),
                    self._E.calScanTime(repr(self.calScanTime)),
                    ),
                self._E.annotation(
                    self._E.sValue(json.dumps(self.annotation)),
                    # annotation attributes:
                    {'issue': 'REALFAST_METADATA',
                        'details': 'Realfast candidate metadata'}
                    ),
                # SdmBuilderMessage attributes:
                {'timestamp': '%.12f' % time.Time.now().mjd,
                    'sender': 'realfast'}
                )

    @property
    def xml(self):
        return etree.tostring(self._root, xml_declaration=True,
                              pretty_print=False, standalone=True)

    @property
    def _url(self):
        query = urlencode({'xml': self.xml})
        url = urlunparse(('https', self.host, self.path, '', query, ''))
        return url

    def send(self):
        response_xml = urlopen(self._url, timeout=60).read()
        self.response = None
        if b'error' in response_xml:
            logger.warn("error in sdmbuilder response xml: {0}"
                        .format(response_xml))
            self.response = response_xml
        else:
            self.response = objectify.fromstring(response_xml,
                                                 parser=_sdmbuilder_parser)

    @property
    def location(self):
        try:
            return str(self.response.result.location)
        except AttributeError:
            logger.warn("No SDM generated.")
            return None


def makesdm(startTime, endTime, datasetId, data, calScanTime=None,
{'fraction_non_alphanumeric': 0.09802777453611856, 'fraction_numerical': 0.0023339946318123466, 'mean_word_length': 2.13345521023766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3709698', 'n_tokens_mistral': 2488, 'n_tokens_neox': 2427, 'n_words': 493}