<commit_before>package eu.modernmt.xml;

import eu.modernmt.io.UTF8Charset;
import org.apache.commons.io.input.BOMInputStream;

import javax.xml.XMLConstants;
import javax.xml.namespace.QName;
import javax.xml.stream.*;
import javax.xml.stream.events.*;
import java.io.*;
import java.nio.charset.Charset;
import java.util.Iterator;

/**
 * Created by davide on 04/07/16.
 */
public class XMLUtils {

    public static String escapeText(String string) {
        StringBuilder builder = null;

        for (int i = 0; i < string.length(); i++) {
            char c = string.charAt(i);

            if (builder == null && (c == '&' || c == '<' || c == '>')) {
                builder = new StringBuilder((int) (string.length() * 1.4));
                builder.append(string, 0, i);
            }

            switch (c) {
                case '&':
                    builder.append("&amp;");
                    break;
                case '<':
                    builder.append("&lt;");
                    break;
                case '>':
                    builder.append("&gt;");
                    break;
                default:
                    if (builder != null)
                        builder.append(c);
                    break;
            }
        }

        return builder == null ? string : builder.toString();
    }

    public static XMLEventReader createEventReader(InputStream stream) throws XMLStreamException {
        XMLInputFactory factory = XMLInputFactory.newInstance();
        return factory.createXMLEventReader(new XMLFixInputStreamReader(new BOMInputStream(stream, false), UTF8Charset.get()));
    }

    public static void closeQuietly(XMLEventReader reader) {
        if (reader != null)
            try {
                reader.close();
            } catch (XMLStreamException e) {
                // Ignore
            }
    }<commit_msg>TMX import now also support UTF-16 files with BOM<commit_after>package eu.modernmt.xml;

import eu.modernmt.io.UTF8Charset;
import org.apache.commons.io.ByteOrderMark;
import org.apache.commons.io.input.BOMInputStream;

import javax.xml.XMLConstants;
import javax.xml.namespace.QName;
import javax.xml.stream.*;
import javax.xml.stream.events.*;
import java.io.*;
import java.nio.charset.Charset;
import java.util.Iterator;

/**
 * Created by davide on 04/07/16.
 */
public class XMLUtils {

    public static String escapeText(String string) {
        StringBuilder builder = null;

        for (int i = 0; i < string.length(); i++) {
            char c = string.charAt(i);

            if (builder == null && (c == '&' || c == '<' || c == '>')) {
                builder = new StringBuilder((int) (string.length() * 1.4));
                builder.append(string, 0, i);
            }

            switch (c) {
                case '&':
                    builder.append("&amp;");
                    break;
                case '<':
                    builder.append("&lt;");
                    break;
                case '>':
                    builder.append("&gt;");
                    break;
                default:
                    if (builder != null)
                        builder.append(c);
                    break;
            }
        }

        return builder == null ? string : builder.toString();
    }

    public static XMLEventReader createEventReader(InputStream stream) throws XMLStreamException {
        Charset charset = UTF8Charset.get();

        BOMInputStream bomStream = new BOMInputStream(stream, false,
                ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE);
        try {
            if (bomStream.hasBOM())
                charset = Charset.forName(bomStream.getBOMCharsetName());
        } catch (IOException e) {
            throw new XMLStreamException(e);
        }

        XMLInputFactory factory = XMLInputFactory.newInstance();
        return factory.createXMLEventReader(new XMLFixInputStreamReader(bomStream, charset));
    }

    public static void closeQuietly(XMLEventReader reader) {
        if (reader != null)
            try {
                reader.close();
            } catch (XMLStreamException e) {
                // Ignore
            }
    }
{'fraction_non_alphanumeric': 0.11792564527587024, 'fraction_numerical': 0.007340753019180677, 'mean_word_length': 1.7059577194106341, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7586219', 'n_tokens_mistral': 1181, 'n_tokens_neox': 1103, 'n_words': 254}