<commit_before># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2006,2007 <NAME> <<EMAIL>>
#
# a little helper to get the proper ElementTree package

import re

try:
    import cElementTree as ET
    import elementtree
    #print "we are on CET"
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
        #print "simply using ET"
    except ImportError:
        """ this seems to be necessary with the python2.5 on the Maemo platform """
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                import sys
                print "no ElementTree module found, critical error"
                sys.exit(0)

#try:
#    from xml.etree import cElementTree as ET
#except ImportError:
#    try:
#        import cElementTree as ET
#    except ImportError:
#        try:
#            from xml.etree import ElementTree as ET
#        except ImportError:
#            try:
#                from elementtree import ElementTree as ET
#            except ImportError:
#                import sys
#                print "no ElementTree module found, critical error"
#               sys.exit(0)

#try:
#    from xml.etree.ElementTree import _ElementInterface
#except ImportError:
#    from elementtree.ElementTree import _ElementInterface

#try:
#    from xml.etree.ElementTree import _encode_entity as old_encode_entity
#    from xml.etree.ElementTree import _escape,_escape_map,_encode,_raise_serialization_error
#except ImportError:
#    from elementtree.ElementTree import _encode_entity as old_encode_entity
#    from elementtree.ElementTree import _escape,_escape_map,_encode,_raise_serialization_error

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def encode_entity(text, pattern=escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        return ''.join(out)
    try:
        return elementtree.ElementTree._encode(pattern.sub(escape_entities, text), 'ascii')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

def new_encode_entity(text, pattern=utf8_escape):<commit_msg>change from a sys.exit and therefore a program abort to a more application friendly ImportError exception when no suitable ElementTree implementation can be found<commit_after># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2006,2007 <NAME> <<EMAIL>>
#
# a little helper to get the proper ElementTree package

import re

try:
    import cElementTree as ET
    import elementtree
    #print "we are on CET"
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
        #print "simply using ET"
    except ImportError:
        """ this seems to be necessary with the python2.5 on the Maemo platform """
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                #print "no ElementTree module found, critical error"
                raise ImportError, "no ElementTree module found, critical error"

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def encode_entity(text, pattern=escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        return ''.join(out)
    try:
        return elementtree.ElementTree._encode(pattern.sub(escape_entities, text), 'ascii')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

def new_encode_entity(text, pattern=utf8_escape):
{'fraction_non_alphanumeric': 0.08546824542518837, 'fraction_numerical': 0.0077502691065662, 'mean_word_length': 2.5384615384615383, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3269294', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1264, 'n_words': 411}