<commit_before>from collections import defaultdict, namedtuple
from datetime import datetime

import blpapi
import pandas as pd
import numpy as np

import tia.util.log as log


SecurityErrorAttrs = ['security', 'source', 'code', 'category', 'message', 'subcategory']
SecurityError = namedtuple('SecurityError', SecurityErrorAttrs)
FieldErrorAttrs = ['security', 'field', 'source', 'code', 'category', 'message', 'subcategory']
FieldError = namedtuple('FieldError', FieldErrorAttrs)

logger = log.get_logger(__name__)

__all__ = ['Terminal']


class XmlHelper(object):
    @staticmethod
    def security_iter(nodearr):
        """ provide a security data iterator by returning a tuple of (Element, SecurityError) which are mutually exclusive """
        assert nodearr.name() == 'securityData' and nodearr.isArray()
        for i in range(nodearr.numValues()):
            node = nodearr.getValue(i)
            err = XmlHelper.get_security_error(node)
            result = (None, err) if err else (node, None)
            yield result

    @staticmethod
    def node_iter(nodearr):
        assert nodearr.isArray()
        for i in range(nodearr.numValues()):
            yield nodearr.getValue(i)

    @staticmethod
    def message_iter(evt):
        """ provide a message iterator which checks for a response error prior to returning """
        for msg in evt:
            if logger.isEnabledFor(log.logging.DEBUG):
                logger.debug(msg.Print)
            if msg.asElement().hasElement('responseError'):
                raise Exception(msg.toString())
            yield msg

    @staticmethod
    def get_sequence_value(node):
        """Convert an element with DataType Sequence to a DataFrame.
        Note this may be a naive implementation as I assume that bulk data is always a table
        """
        assert node.datatype() == 15
        data = defaultdict(list)
        cols = []
        for i in range(node.numValues()):
            row = node.getValue(i)
            if i == 0:  # Get the ordered cols and assume they are constant
                cols = [str(row.getElement(_).name()) for _ in range(row.numElements())]

            for cidx in range(row.numElements()):
                col = row.getElement(cidx)
                data[str(col.name())].append(XmlHelper.as_value(col))
        return pd.DataFrame(data, columns=cols)

    @staticmethod
    def as_value(ele):
        """ convert the specified element as a python value """
        dtype = ele.datatype()
        # print '%s = %s' % (ele.name(), dtype)
        if dtype in (1, 2, 3, 4, 5, 6, 7, 9, 12):
            # BOOL, CHAR, BYTE, INT32, INT64, FLOAT32, FLOAT64, BYTEARRAY, DECIMAL)
            return ele.getValue()
        elif dtype == 8:  # String
            val = ele.getValue()
            """
            if val:
                # us centric :)
                val = val.encode('ascii', 'replace')
            """
            return str(val)
        elif dtype == 10:  # Date
            v = ele.getValue()
            return datetime(year=v.year, month=v.month, day=v.day) if v else np.nan
        elif dtype == 11:  # Time
            v = ele.getValue()
            return datetime(hour=v.hour, minute=v.minute, second=v.second).time() if v else np.nan
        elif dtype == 13:  # Datetime
            v = ele.getValue()
            return v
        elif dtype == 14:  # Enumeration
            # raise NotImplementedError('ENUMERATION data type needs implemented')
            return str(ele.getValue())
        elif dtype == 16:  # Choice
            raise NotImplementedError('CHOICE data type needs implemented')
        elif dtype == 15:  # SEQUENCE
            return XmlHelper.get_sequence_value(ele)
        else:
            raise NotImplementedError('Unexpected data type %s. Check documentation' % dtype)

    @staticmethod
    def get_child_value(parent, name, allow_missing=0):
        """ return the value of the child element with name in the parent Element """
        if not parent.hasElement(name):
            if allow_missing:
                return np.nan
            else:
                raise Exception('failed to find child element %s in parent' % name)
        else:
            return XmlHelper.as_value(parent.getElement(name))

    @staticmethod
    def get_child_values(parent, names):
        """ return a list of values for the specified child fields. If field not in Element then replace with nan. """
        vals = []
        for name in names:
            if parent.hasElement(name):
                vals.append(XmlHelper.as_value(parent.getElement(name)))<commit_msg>fixed bug when no value present for date node<commit_after>from collections import defaultdict, namedtuple
from datetime import datetime

import blpapi
import pandas as pd
import numpy as np

import tia.util.log as log


SecurityErrorAttrs = ['security', 'source', 'code', 'category', 'message', 'subcategory']
SecurityError = namedtuple('SecurityError', SecurityErrorAttrs)
FieldErrorAttrs = ['security', 'field', 'source', 'code', 'category', 'message', 'subcategory']
FieldError = namedtuple('FieldError', FieldErrorAttrs)

logger = log.get_logger(__name__)

__all__ = ['Terminal']


class XmlHelper(object):
    @staticmethod
    def security_iter(nodearr):
        """ provide a security data iterator by returning a tuple of (Element, SecurityError) which are mutually exclusive """
        assert nodearr.name() == 'securityData' and nodearr.isArray()
        for i in range(nodearr.numValues()):
            node = nodearr.getValue(i)
            err = XmlHelper.get_security_error(node)
            result = (None, err) if err else (node, None)
            yield result

    @staticmethod
    def node_iter(nodearr):
        assert nodearr.isArray()
        for i in range(nodearr.numValues()):
            yield nodearr.getValue(i)

    @staticmethod
    def message_iter(evt):
        """ provide a message iterator which checks for a response error prior to returning """
        for msg in evt:
            if logger.isEnabledFor(log.logging.DEBUG):
                logger.debug(msg.Print)
            if msg.asElement().hasElement('responseError'):
                raise Exception(msg.toString())
            yield msg

    @staticmethod
    def get_sequence_value(node):
        """Convert an element with DataType Sequence to a DataFrame.
        Note this may be a naive implementation as I assume that bulk data is always a table
        """
        assert node.datatype() == 15
        data = defaultdict(list)
        cols = []
        for i in range(node.numValues()):
            row = node.getValue(i)
            if i == 0:  # Get the ordered cols and assume they are constant
                cols = [str(row.getElement(_).name()) for _ in range(row.numElements())]

            for cidx in range(row.numElements()):
                col = row.getElement(cidx)
                data[str(col.name())].append(XmlHelper.as_value(col))
        return pd.DataFrame(data, columns=cols)

    @staticmethod
    def as_value(ele):
        """ convert the specified element as a python value """
        dtype = ele.datatype()
        # print '%s = %s' % (ele.name(), dtype)
        if dtype in (1, 2, 3, 4, 5, 6, 7, 9, 12):
            # BOOL, CHAR, BYTE, INT32, INT64, FLOAT32, FLOAT64, BYTEARRAY, DECIMAL)
            return ele.getValue()
        elif dtype == 8:  # String
            val = ele.getValue()
            """
            if val:
                # us centric :)
                val = val.encode('ascii', 'replace')
            """
            return str(val)
        elif dtype == 10:  # Date
            if ele.isNull():
                return pd.NaT
            else:
                v = ele.getValue()
                return datetime(year=v.year, month=v.month, day=v.day) if v else pd.NaT
        elif dtype == 11:  # Time
            if ele.isNull():
                return pd.NaT
            else:
                v = ele.getValue()
                return datetime(hour=v.hour, minute=v.minute, second=v.second).time() if v else np.nan
        elif dtype == 13:  # Datetime
            if ele.isNull():
                return pd.NaT
            else:
                v = ele.getValue()
                return v
        elif dtype == 14:  # Enumeration
            # raise NotImplementedError('ENUMERATION data type needs implemented')
            return str(ele.getValue())
        elif dtype == 16:  # Choice
            raise NotImplementedError('CHOICE data type needs implemented')
        elif dtype == 15:  # SEQUENCE
            return XmlHelper.get_sequence_value(ele)
        else:
            raise NotImplementedError('Unexpected data type %s. Check documentation' % dtype)

    @staticmethod
    def get_child_value(parent, name, allow_missing=0):
        """ return the value of the child element with name in the parent Element """
        if not parent.hasElement(name):
            if allow_missing:
                return np.nan
            else:
                raise Exception('failed to find child element %s in parent' % name)
        else:
            return XmlHelper.as_value(parent.getElement(name))

    @staticmethod
    def get_child_values(parent, names):
        """ return a list of values for the specified child fields. If field not in Element then replace with nan. """
        vals = []
        for name in names:
            if parent.hasElement(name):
                vals.append(XmlHelper.as_value(parent.getElement(name)))
{'fraction_non_alphanumeric': 0.10226196738558653, 'fraction_numerical': 0.007364544976328248, 'mean_word_length': 2.147682119205298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3050338', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2534, 'n_words': 820}