<commit_before># Copyright 2011-2015 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""The **splunklib.data** module reads the responses from splunkd in Atom Feed 
format, which is the format used by most of the REST API.
"""

from __future__ import absolute_import
from xml.etree.ElementTree import XML
from splunklib import six

__all__ = ["load"]

# LNAME refers to element names without namespaces; XNAME is the same
# name, but with an XML namespace.
LNAME_DICT = "dict"
LNAME_ITEM = "item"
LNAME_KEY = "key"
LNAME_LIST = "list"

XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s"
XNAME_DICT = XNAMEF_REST % LNAME_DICT
XNAME_ITEM = XNAMEF_REST % LNAME_ITEM
XNAME_KEY = XNAMEF_REST % LNAME_KEY
XNAME_LIST = XNAMEF_REST % LNAME_LIST

# Some responses don't use namespaces (eg: search/parse) so we look for
# both the extended and local versions of the following names.

def isdict(name):
    return name == XNAME_DICT or name == LNAME_DICT

def isitem(name):
    return name == XNAME_ITEM or name == LNAME_ITEM

def iskey(name):
    return name == XNAME_KEY or name == LNAME_KEY

def islist(name):
    return name == XNAME_LIST or name == LNAME_LIST

def hasattrs(element):
    return len(element.attrib) > 0

def localname(xname):
    rcurly = xname.find('}')
    return xname if rcurly == -1 else xname[rcurly+1:]

def load(text, match=None):
    """This function reads a string that contains the XML of an Atom Feed, then 
    returns the 
    data in a native Python structure (a ``dict`` or ``list``). If you also 
    provide a tag name or path to match, only the matching sub-elements are 
    loaded.

    :param text: The XML text to load.
    :type text: ``string``
    :param match: A tag name or path to match (optional).
    :type match: ``string``
    """
    if text is None: return None
    text = text.strip()
    if len(text) == 0: return None
    nametable = {
        'namespaces': [],
        'names': {}
    }

    if isinstance(text, unicode):
        text = text.encode('utf-8')

    root = XML(text)
    items = [root] if match is None else root.findall(match)
    count = len(items)
    if count == 0: 
        return None
    elif count == 1: 
        return load_root(items[0], nametable)
    else:
        return [load_root(item, nametable) for item in items]

# Load the attributes of the given element.
def load_attrs(element):
    if not hasattrs(element): return None<commit_msg>Fix XML responses to not throw errors for unicode characters<commit_after># Copyright 2011-2015 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""The **splunklib.data** module reads the responses from splunkd in Atom Feed 
format, which is the format used by most of the REST API.
"""

from __future__ import absolute_import
import sys
from xml.etree.ElementTree import XML
from splunklib import six

__all__ = ["load"]

# LNAME refers to element names without namespaces; XNAME is the same
# name, but with an XML namespace.
LNAME_DICT = "dict"
LNAME_ITEM = "item"
LNAME_KEY = "key"
LNAME_LIST = "list"

XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s"
XNAME_DICT = XNAMEF_REST % LNAME_DICT
XNAME_ITEM = XNAMEF_REST % LNAME_ITEM
XNAME_KEY = XNAMEF_REST % LNAME_KEY
XNAME_LIST = XNAMEF_REST % LNAME_LIST

# Some responses don't use namespaces (eg: search/parse) so we look for
# both the extended and local versions of the following names.

def isdict(name):
    return name == XNAME_DICT or name == LNAME_DICT

def isitem(name):
    return name == XNAME_ITEM or name == LNAME_ITEM

def iskey(name):
    return name == XNAME_KEY or name == LNAME_KEY

def islist(name):
    return name == XNAME_LIST or name == LNAME_LIST

def hasattrs(element):
    return len(element.attrib) > 0

def localname(xname):
    rcurly = xname.find('}')
    return xname if rcurly == -1 else xname[rcurly+1:]

def load(text, match=None):
    """This function reads a string that contains the XML of an Atom Feed, then 
    returns the 
    data in a native Python structure (a ``dict`` or ``list``). If you also 
    provide a tag name or path to match, only the matching sub-elements are 
    loaded.

    :param text: The XML text to load.
    :type text: ``string``
    :param match: A tag name or path to match (optional).
    :type match: ``string``
    """
    if text is None: return None
    text = text.strip()
    if len(text) == 0: return None
    nametable = {
        'namespaces': [],
        'names': {}
    }

    # Convert to unicode encoding in only python 2 for xml parser
    if sys.version_info < (3, 0, 0):
        if isinstance(text, unicode):
            text = text.encode('utf-8')

    root = XML(text)
    items = [root] if match is None else root.findall(match)
    count = len(items)
    if count == 0: 
        return None
    elif count == 1: 
        return load_root(items[0], nametable)
    else:
        return [load_root(item, nametable) for item in items]

# Load the attributes of the given element.
def load_attrs(element):
    if not hasattrs(element): return None
{'fraction_non_alphanumeric': 0.09397310310476507, 'fraction_numerical': 0.0073053295699817365, 'mean_word_length': 3.4392041267501843, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1601946', 'n_tokens_mistral': 1912, 'n_tokens_neox': 1834, 'n_words': 752}