<commit_before>#
# (C) Copyright 2003 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License Version
# 2.1 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

"""Utility functions for the pyxmpp package."""

from types import UnicodeType,StringType
import re

def to_utf8(s):
    """
    Convevert `s` to UTF-8 if it is Unicode, leave unchanged 
    if it is string or None and convert to string overwise
    """
    if s is None:
        return None
    elif type(s) is UnicodeType:
        return s.encode("utf-8")
    else:
        return str(s)

def from_utf8(s):
    """
    Convert `s` to Unicode or leave unchanged if it is None.
    
    Regular strings are assumed to be UTF-8 encoded
    """
    if s is None:
        return None
    elif type(s) is UnicodeType:
        return s
    elif type(s) is StringType:
        return unicode(s,"utf-8")
    else:
        return unicode(s)

evil_characters_re=re.compile(r"[\000-\010\013\014\016-\037]",re.UNICODE)
utf8_replacement_char=u"\ufffd".encode("utf-8")
def remove_evil_characters(s):
    """Remove control characters (not allowed in XML) from a string."""
    if type(s) is UnicodeType:
        return evil_characters_re.sub(u"\ufffd",s)
    else:
        return evil_characters_re.sub(utf8_replacement_char,s)

def get_node_ns(node):
    """Return namespace of the XML `node` or None if namespace is not set."""
    try:
        return node.ns()
    except libxml2.treeError:
        return None


def get_node_ns_uri(node):
    """Return namespace URI of the XML `node` or None if namespace is not set."""
    ns=get_node_ns(node)
    if ns:
        return ns.getContent()
    else:
        return None
    
# vi: sts=4 et sw=4
<commit_msg>- missing 'import libxml2' added<commit_after>#
# (C) Copyright 2003 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License Version
# 2.1 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

"""Utility functions for the pyxmpp package."""

from types import UnicodeType,StringType
import re
import libxml2

def to_utf8(s):
    """
    Convevert `s` to UTF-8 if it is Unicode, leave unchanged 
    if it is string or None and convert to string overwise
    """
    if s is None:
        return None
    elif type(s) is UnicodeType:
        return s.encode("utf-8")
    else:
        return str(s)

def from_utf8(s):
    """
    Convert `s` to Unicode or leave unchanged if it is None.
    
    Regular strings are assumed to be UTF-8 encoded
    """
    if s is None:
        return None
    elif type(s) is UnicodeType:
        return s
    elif type(s) is StringType:
        return unicode(s,"utf-8")
    else:
        return unicode(s)

evil_characters_re=re.compile(r"[\000-\010\013\014\016-\037]",re.UNICODE)
utf8_replacement_char=u"\ufffd".encode("utf-8")
def remove_evil_characters(s):
    """Remove control characters (not allowed in XML) from a string."""
    if type(s) is UnicodeType:
        return evil_characters_re.sub(u"\ufffd",s)
    else:
        return evil_characters_re.sub(utf8_replacement_char,s)

def get_node_ns(node):
    """Return namespace of the XML `node` or None if namespace is not set."""
    try:
        return node.ns()
    except libxml2.treeError:
        return None


def get_node_ns_uri(node):
    """Return namespace URI of the XML `node` or None if namespace is not set."""
    ns=get_node_ns(node)
    if ns:
        return ns.getContent()
    else:
        return None
    
# vi: sts=4 et sw=4

{'fraction_non_alphanumeric': 0.09072847682119205, 'fraction_numerical': 0.019867549668874173, 'mean_word_length': 3.0600358422939067, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6091210', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1356, 'n_words': 554}