<commit_before>#!/usr/bin/env python2

import sys
from sys import argv, stdout, stderr
import codecs, locale
import os
import xml.dom.minidom

if sys.version_info[0] < 3:
    stdout = codecs.getwriter('utf-8')(stdout)

NS_XI = 'http://www.w3.org/2001/XInclude'

def xincludate(dom, base, dropns = []):
    remove_attrs = []
    for i in range(dom.documentElement.attributes.length):
        attr = dom.documentElement.attributes.item(i)
        if attr.prefix == 'xmlns':
            if attr.localName in dropns:
                remove_attrs.append(attr)
            else:
                dropns.append(attr.localName)
    for attr in remove_attrs:
        dom.documentElement.removeAttributeNode(attr)
    for include in dom.getElementsByTagNameNS(NS_XI, 'include'):
        href = include.getAttribute('href')
        # FIXME: assumes Unixy paths
        filename = os.path.join(os.path.dirname(base), href)
        subdom = xml.dom.minidom.parse(filename)
        xincludate(subdom, filename, dropns)
        if './' in href:
            subdom.documentElement.setAttribute('xml:base', href)
        include.parentNode.replaceChild(subdom.documentElement, include)

if __name__ == '__main__':
    argv = argv[1:]
    dom = xml.dom.minidom.parse(argv[0])
    xincludate(dom, argv[0])

    if sys.version_info[0] >= 3:
        xml = dom.toxml(encoding=None)
    else:
        xml = dom.toxml()

    stdout.write(xml)
    stdout.write('\n')<commit_msg>xincludator.py: Fix behaviour on LANG=C<commit_after>#!/usr/bin/env python2

import sys
from sys import argv, stdout, stderr
import codecs, locale
import os
import xml.dom.minidom

if sys.version_info[0] < 3:
    stdout = codecs.getwriter('utf-8')(stdout)

NS_XI = 'http://www.w3.org/2001/XInclude'

def xincludate(dom, base, dropns = []):
    remove_attrs = []
    for i in range(dom.documentElement.attributes.length):
        attr = dom.documentElement.attributes.item(i)
        if attr.prefix == 'xmlns':
            if attr.localName in dropns:
                remove_attrs.append(attr)
            else:
                dropns.append(attr.localName)
    for attr in remove_attrs:
        dom.documentElement.removeAttributeNode(attr)
    for include in dom.getElementsByTagNameNS(NS_XI, 'include'):
        href = include.getAttribute('href')
        # FIXME: assumes Unixy paths
        filename = os.path.join(os.path.dirname(base), href)
        subdom = xml.dom.minidom.parse(filename)
        xincludate(subdom, filename, dropns)
        if './' in href:
            subdom.documentElement.setAttribute('xml:base', href)
        include.parentNode.replaceChild(subdom.documentElement, include)

if __name__ == '__main__':
    argv = argv[1:]
    dom = xml.dom.minidom.parse(argv[0])
    xincludate(dom, argv[0])

    if sys.version_info[0] >= 3:
        xml = dom.toxml('utf-8')
        stdout.buffer.write(xml)
        stdout.buffer.write(b'\n')
    else:
        xml = dom.toxml()
        stdout.write(xml)
        stdout.write('\n')
{'fraction_non_alphanumeric': 0.12670890296765588, 'fraction_numerical': 0.009669889963321108, 'mean_word_length': 2.717472118959108, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 21}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6137743', 'n_tokens_mistral': 1005, 'n_tokens_neox': 979, 'n_words': 195}