<commit_before># vim:sw=4:et

import sys
import xml.sax.handler
from xml.sax.saxutils import escape, quoteattr

from past.builtins import basestring

# See whether the xmlcharrefreplace error handler is
# supported
try:
    from codecs import xmlcharrefreplace_errors
    _error_handling = "xmlcharrefreplace"
    del xmlcharrefreplace_errors
except ImportError:
    _error_handling = "strict"

class XMLWriter(xml.sax.handler.ContentHandler):

    def __init__(self, out=None, encoding=None):
        if out is None:
            out = sys.stdout
        xml.sax.handler.ContentHandler.__init__(self)
        self._out = out
        self._ns_contexts = [{}] # contains uri -> prefix dicts
        self._current_context = self._ns_contexts[-1]
        self._undeclared_ns_maps = []
        self._encoding = encoding or sys.getdefaultencoding()

        self._in_cdata = False
        self._in_start_tag = False
        self._next_newline = False

    def _write(self, text, start_tag=False, end_tag=False):
        """
        Write data. Tags should not be escaped. They should be marked
        by setting either ``start_tag`` or ``end_tag`` to ``True``.
        Only the tag should be marked this way. Other stuff, such as
        namespaces and attributes can be written directly to the file.
        """
        if not isinstance(text, unicode):
            text = text.decode(self._encoding, _error_handling)

        if self._next_newline:
            self._out.write(u'\n')
            self._next_newline = False

        if start_tag and not self._in_start_tag:
            self._in_start_tag = True
            self._out.write(u'<')
        elif start_tag and self._in_start_tag:
            self._out.write(u'>')<commit_msg>Type unicode becomes str in Python3<commit_after># vim:sw=4:et

import sys
import xml.sax.handler
from builtins import str
from xml.sax.saxutils import escape, quoteattr

# See whether the xmlcharrefreplace error handler is
# supported
try:
    from codecs import xmlcharrefreplace_errors
    _error_handling = "xmlcharrefreplace"
    del xmlcharrefreplace_errors
except ImportError:
    _error_handling = "strict"


class XMLWriter(xml.sax.handler.ContentHandler):

    def __init__(self, out=None, encoding=None):
        if out is None:
            out = sys.stdout
        xml.sax.handler.ContentHandler.__init__(self)
        self._out = out
        self._ns_contexts = [{}] # contains uri -> prefix dicts
        self._current_context = self._ns_contexts[-1]
        self._undeclared_ns_maps = []
        self._encoding = encoding or sys.getdefaultencoding()

        self._in_cdata = False
        self._in_start_tag = False
        self._next_newline = False

    def _write(self, text, start_tag=False, end_tag=False):
        """
        Write data. Tags should not be escaped. They should be marked
        by setting either ``start_tag`` or ``end_tag`` to ``True``.
        Only the tag should be marked this way. Other stuff, such as
        namespaces and attributes can be written directly to the file.
        """
        if not isinstance(text, str):
            text = text.decode(self._encoding, _error_handling)

        if self._next_newline:
            self._out.write(u'\n')
            self._next_newline = False

        if start_tag and not self._in_start_tag:
            self._in_start_tag = True
            self._out.write(u'<')
        elif start_tag and self._in_start_tag:
            self._out.write(u'>')
{'fraction_non_alphanumeric': 0.08606793321819228, 'fraction_numerical': 0.0014392630972941854, 'mean_word_length': 2.5861713106295148, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3709193', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1009, 'n_words': 284}