<commit_before># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import contextlib
import functools
import keyword

import lxml.etree

from mixbox.entities import Entity, EntityList
import mixbox.xml

import stix

# relative
from . import dates


CDATA_START = "<![CDATA["
CDATA_END = "]]>"
CONFLICTING_NAMES = keyword.kwlist + ['id', 'type', 'range']


@contextlib.contextmanager
def ignored(*exceptions):
    """Allows you to ignore exceptions cleanly using context managers. This
    exists in Python 3.

    """
    try:
        yield
    except exceptions:
        pass


def raise_warnings(func):
    """Function decorator that causes all Python warnings to be raised as
    exceptions in the wrapped function.

    Example:
        >>> @raise_warnings
        >>> def foo():
        >>>     warnings.warn("this will raise an exception")

    """
    @functools.wraps(func)
    def inner(*args, **kwargs):
        with warnings.catch_warnings():
            warnings.simplefilter('error')
            return func(*args, **kwargs)
    return inner


def silence_warnings(func):
    """Function decorator that silences/ignores all Python warnings in the
    wrapped function.

    Example:
        >>> @silence_warnings
        >>> def foo():
        >>>     warnings.warn("this will not appear")

    """
    @functools.wraps(func)
    def inner(*args, **kwargs):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always')
            return func(*args, **kwargs)
    return inner


def is_cdata(text):
    """Returns ``True`` if `text` contains a CDATA block.

    Example:
        >>> is_cdata("<![CDATA[Foo]]>")
        True
        >>> is_cdata("NOPE")
        False

    """
    if not text:
        return False

    return CDATA_START in text


def strip_cdata(text):
    """Removes all CDATA blocks from `text` if it contains them.

    Note:
        If the function contains escaped XML characters outside of a
        CDATA block, they will be unescaped.

    Args:
        A string containing one or more CDATA blocks.

    Returns:
        An XML unescaped string with CDATA block qualifiers removed.

    """
    if not is_cdata(text):
        return text

    xml = "<e>{0}</e>".format(text)
    node = lxml.etree.fromstring(xml)
    return node.text


def cdata(text):
    """Wraps the input `text` in a ``<![CDATA[ ]]>`` block.

    If the text contains CDATA sections already, they are stripped and replaced
    by the application of an outer-most CDATA block.

    Args:
        text: A string to wrap in a CDATA block.

    Returns:
        The `text` value wrapped in ``<![CDATA[]]>``

    """
    if not text:
        return text

    if is_cdata(text):
        text = strip_cdata(text)

    escaped = "{0}{1}{2}".format(CDATA_START, text, CDATA_END)
    return escaped


def is_stix(entity):
    """Returns true if `entity` is an instance of :class:`.Entity`."""
    return isinstance(entity, stix.Entity)


def is_cybox(entity):
    """Returns true if `entity` is an instance of
    :class:`mixbox.entities.Entity`.
    """
    # TODO: once all entities subclass from mixbox.entities.Entity, how do we
    # do this?
    return isinstance(entity, Entity)
<commit_msg>is_cybox now accounts for the fact that stix entites are mixbox entities<commit_after># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import contextlib
import functools
import keyword

import lxml.etree

from mixbox.entities import Entity, EntityList
import mixbox.xml

import stix

# relative
from . import dates


CDATA_START = "<![CDATA["
CDATA_END = "]]>"
CONFLICTING_NAMES = keyword.kwlist + ['id', 'type', 'range']


@contextlib.contextmanager
def ignored(*exceptions):
    """Allows you to ignore exceptions cleanly using context managers. This
    exists in Python 3.

    """
    try:
        yield
    except exceptions:
        pass


def raise_warnings(func):
    """Function decorator that causes all Python warnings to be raised as
    exceptions in the wrapped function.

    Example:
        >>> @raise_warnings
        >>> def foo():
        >>>     warnings.warn("this will raise an exception")

    """
    @functools.wraps(func)
    def inner(*args, **kwargs):
        with warnings.catch_warnings():
            warnings.simplefilter('error')
            return func(*args, **kwargs)
    return inner


def silence_warnings(func):
    """Function decorator that silences/ignores all Python warnings in the
    wrapped function.

    Example:
        >>> @silence_warnings
        >>> def foo():
        >>>     warnings.warn("this will not appear")

    """
    @functools.wraps(func)
    def inner(*args, **kwargs):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always')
            return func(*args, **kwargs)
    return inner


def is_cdata(text):
    """Returns ``True`` if `text` contains a CDATA block.

    Example:
        >>> is_cdata("<![CDATA[Foo]]>")
        True
        >>> is_cdata("NOPE")
        False

    """
    if not text:
        return False

    return CDATA_START in text


def strip_cdata(text):
    """Removes all CDATA blocks from `text` if it contains them.

    Note:
        If the function contains escaped XML characters outside of a
        CDATA block, they will be unescaped.

    Args:
        A string containing one or more CDATA blocks.

    Returns:
        An XML unescaped string with CDATA block qualifiers removed.

    """
    if not is_cdata(text):
        return text

    xml = "<e>{0}</e>".format(text)
    node = lxml.etree.fromstring(xml)
    return node.text


def cdata(text):
    """Wraps the input `text` in a ``<![CDATA[ ]]>`` block.

    If the text contains CDATA sections already, they are stripped and replaced
    by the application of an outer-most CDATA block.

    Args:
        text: A string to wrap in a CDATA block.

    Returns:
        The `text` value wrapped in ``<![CDATA[]]>``

    """
    if not text:
        return text

    if is_cdata(text):
        text = strip_cdata(text)

    escaped = "{0}{1}{2}".format(CDATA_START, text, CDATA_END)
    return escaped


def is_stix(entity):
    """Returns true if `entity` is an instance of :class:`.Entity`."""
    return isinstance(entity, stix.Entity)


def is_cybox(entity):
    """Returns true if `entity` is a Cybox object"""
    try:
        return entity.__module__.startswith("cybox.")
    except AttributeError:
        return False

{'fraction_non_alphanumeric': 0.10991018419850815, 'fraction_numerical': 0.002740143096361699, 'mean_word_length': 2.674496644295302, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5487874', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1903, 'n_words': 648}