<commit_before>/*
 *  Copyright 2013, 2014 Deutsche Nationalbibliothek
 *
 *  Licensed under the Apache License, Version 2.0 the "License";
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.culturegraph.mf.stream.converter.xml;

import java.io.IOException;
import java.io.Reader;

import org.culturegraph.mf.exceptions.MetafactureException;
import org.culturegraph.mf.framework.DefaultObjectPipe;
import org.culturegraph.mf.framework.XmlReceiver;
import org.culturegraph.mf.framework.annotations.Description;
import org.culturegraph.mf.framework.annotations.In;
import org.culturegraph.mf.framework.annotations.Out;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xml.sax.XMLFilter;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLFilterImpl;
import org.xml.sax.helpers.XMLReaderFactory;


/**
 * Reads an XML file and passes the XML events to a receiver.
 * 
 * @author <NAME>
 * 
 */
@Description("Reads an XML file and passes the XML events to a receiver.")
@In(Reader.class)
@Out(XmlReceiver.class)
public final class XmlDecoder 
		extends DefaultObjectPipe<Reader, XmlReceiver> {
	
	private static final String SAX_PROPERTY_LEXICAL_HANDLER = "http://xml.org/sax/properties/lexical-handler";

	private final XMLFilter xmlPreFilter;
	private final XMLFilter xmlFilter;

	public XmlDecoder() {
		super();
		try {
			final XMLReader saxReader = XMLReaderFactory.createXMLReader();
			xmlPreFilter = new XmlFilterEntityImpl(saxReader);
			xmlFilter = new XMLFilterImpl(xmlPreFilter);
		} catch (SAXException e) {
			throw new MetafactureException(e);
		}
	}

	@Override
	public void process(final Reader reader) {
		try {
			xmlFilter.parse(new InputSource(reader));
		} catch (IOException e) {
			throw new MetafactureException(e);
		} catch (SAXException e) {
			throw new MetafactureException(e);
		}
	}

	@Override
	protected void onSetReceiver() {
		xmlFilter.setContentHandler(getReceiver());
		xmlFilter.setDTDHandler(getReceiver());
		xmlFilter.setEntityResolver(getReceiver());
		xmlFilter.setErrorHandler(getReceiver());<commit_msg>added xerces features for skipping builtin-refs<commit_after>/*
 *  Copyright 2013, 2014 Deutsche Nationalbibliothek
 *
 *  Licensed under the Apache License, Version 2.0 the "License";
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.culturegraph.mf.stream.converter.xml;

import java.io.IOException;
import java.io.Reader;

import org.culturegraph.mf.exceptions.MetafactureException;
import org.culturegraph.mf.framework.DefaultObjectPipe;
import org.culturegraph.mf.framework.XmlReceiver;
import org.culturegraph.mf.framework.annotations.Description;
import org.culturegraph.mf.framework.annotations.In;
import org.culturegraph.mf.framework.annotations.Out;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xml.sax.XMLFilter;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLFilterImpl;
import org.xml.sax.helpers.XMLReaderFactory;


/**
 * Reads an XML file and passes the XML events to a receiver.
 * 
 * @author <NAME>
 * 
 */
@Description("Reads an XML file and passes the XML events to a receiver.")
@In(Reader.class)
@Out(XmlReceiver.class)
public final class XmlDecoder 
		extends DefaultObjectPipe<Reader, XmlReceiver> {
	
	private static final String SAX_PROPERTY_LEXICAL_HANDLER = "http://xml.org/sax/properties/lexical-handler";
	private static final String SAX_FEATURE_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";
	private static final String XERCES_FEATURES_NOTIFY_BUILTIN_REFS = "http://apache.org/xml/features/scanner/notify-builtin-refs";

	private final XMLFilter xmlPreFilter;
	private final XMLFilter xmlFilter;

	public XmlDecoder() {
		super();
		try {
			final XMLReader saxReader = XMLReaderFactory.createXMLReader();
			saxReader.setFeature(SAX_FEATURE_EXTERNAL_GENERAL_ENTITIES, false);
			saxReader.setFeature(XERCES_FEATURES_NOTIFY_BUILTIN_REFS, false);
			xmlPreFilter = new XmlFilterEntityImpl(saxReader);
			xmlFilter = new XMLFilterImpl(xmlPreFilter);
		} catch (SAXException e) {
			throw new MetafactureException(e);
		}
	}

	@Override
	public void process(final Reader reader) {
		try {
			xmlFilter.parse(new InputSource(reader));
		} catch (IOException e) {
			throw new MetafactureException(e);
		} catch (SAXException e) {
			throw new MetafactureException(e);
		}
	}

	@Override
	protected void onSetReceiver() {
		xmlFilter.setContentHandler(getReceiver());
		xmlFilter.setDTDHandler(getReceiver());
		xmlFilter.setEntityResolver(getReceiver());
		xmlFilter.setErrorHandler(getReceiver());
{'fraction_non_alphanumeric': 0.09475346552026671, 'fraction_numerical': 0.004211265134234076, 'mean_word_length': 4.949895615866389, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 42}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '830618', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1708, 'n_words': 452}