<commit_before>/*
 * Copyright 2011-2015 The Trustees of the University of Pennsylvania
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.upennlib.paralleltransformer;

import edu.upennlib.xmlutils.JoiningXMLFilter;
import edu.upennlib.xmlutils.LoggingErrorListener;
import edu.upennlib.xmlutils.SimpleLocalAbsoluteSAXXPath;
import edu.upennlib.xmlutils.SplittingXMLFilter;
import edu.upennlib.xmlutils.UnboundedContentHandlerBuffer;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringWriter;
import java.util.EnumMap;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.URIResolver;
import javax.xml.transform.sax.SAXResult;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import net.sf.saxon.Controller;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xml.sax.XMLReader;
import org.xml.sax.ContentHandler;
import org.xml.sax.ext.LexicalHandler;
import org.xml.sax.helpers.XMLFilterImpl;

/**
 *
 * @author michael
 */
public class TransformingXMLFilter extends JoiningXMLFilter {

    private SplittingXMLFilter splitter;
    private SplittingXMLFilter privateSplitter;
    private File stylesheet;
    private Templates stylesheetTemplates;
    public static final String TRANSFORMER_FACTORY_CLASS_NAME = "net.sf.saxon.TransformerFactoryImpl";
    private static final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactory.newInstance(TRANSFORMER_FACTORY_CLASS_NAME, null);
    private static final String USAGE = "USAGE: command inputFile stylesheet outputFile";
    private static final Logger logger = Logger.getLogger(TransformingXMLFilter.class);

    private static void printUsage(String errorMessage, int exitCode) {
        System.err.println(errorMessage);
        System.err.println(USAGE);
        System.exit(exitCode);
    }

    public static void main(String[] args) throws SAXException, ParserConfigurationException, TransformerConfigurationException, TransformerException, IOException {
        BasicConfigurator.configure();
        if (args.length != 3) {
            printUsage("wrong number of arguments: "+args.length, -1);
        }

        File inputFile = new File(args[0]);
        File stylesheet = new File(args[1]);
        File outputFile = new File(args[2]);

        TransformingXMLFilter txf = new TransformingXMLFilter();
        try {
            txf.setStylesheet(stylesheet);
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        InputSource inputSource = null;
        try {
            inputSource = new InputSource(new BufferedInputStream(new FileInputStream(inputFile)));
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        BufferedOutputStream bos = null;
        try {
            bos = new BufferedOutputStream(new FileOutputStream(outputFile));
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        Transformer t = stf.newTransformer();
        txf.configureOutputTransformer((Controller) t);

        long start = System.currentTimeMillis();
        t.transform(new SAXSource(txf, inputSource), new StreamResult(bos));
        bos.close();
        System.out.println("transformation complete; duration: " + (System.currentTimeMillis() - start)+ " ms");
    }

    public TransformingXMLFilter() throws ParserConfigurationException, SAXException {
        privateSplitter = new SplittingXMLFilter();
        super.setParent(privateSplitter);
        setUpstreamSplittingFilter(privateSplitter);

        SAXParserFactory spf = SAXParserFactory.newInstance();
        spf.setNamespaceAware(true);
        XMLReader reader = spf.newSAXParser().getXMLReader();
        privateSplitter.setParent(reader);
    }

    @Override
    public void setParent(XMLReader parent) {
        super.setParent(parent);
        privateSplitter = null;
    }

    public void setStreamingParent(XMLReader parent) {
        if (privateSplitter == null) {
            privateSplitter = new SplittingXMLFilter();
            super.setParent(privateSplitter);
            setUpstreamSplittingFilter(privateSplitter);
        }
        privateSplitter.setParent(parent);
    }

    public XMLReader getStreamingParent() {
        if (privateSplitter == null) {
            return null;
        } else {
            return privateSplitter.getParent();
        }
    }

    public void setStylesheet(File stylesheet) throws TransformerConfigurationException, FileNotFoundException {
        if (stylesheet == null) {
            stylesheetTemplates = null;
        } else {
            stylesheetTemplates = stf.newTemplates(new StreamSource(stylesheet));
        }
        this.stylesheet = stylesheet;
    }

    public File getStylesheet() {
        return stylesheet;
    }

    public void configureOutputTransformer(Controller out) {
        if (stylesheet != null) {
            try {
                Controller c;
                c = (Controller) stylesheetTemplates.newTransformer();
                out.getExecutable().setCharacterMapIndex(c.getExecutable().getCharacterMapIndex());
                out.setOutputProperties(c.getOutputProperties());

            } catch (TransformerConfigurationException ex) {
                throw new RuntimeException(ex);
            }
        }
    }

    public final void setUpstreamSplittingFilter(SplittingXMLFilter splitter) {
        this.splitter = splitter;<commit_msg>add ability to set an InputStream for the stylesheet instead of only a File<commit_after>/*
 * Copyright 2011-2015 The Trustees of the University of Pennsylvania
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.upennlib.paralleltransformer;

import edu.upennlib.xmlutils.JoiningXMLFilter;
import edu.upennlib.xmlutils.LoggingErrorListener;
import edu.upennlib.xmlutils.SimpleLocalAbsoluteSAXXPath;
import edu.upennlib.xmlutils.SplittingXMLFilter;
import edu.upennlib.xmlutils.UnboundedContentHandlerBuffer;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.StringWriter;
import java.util.EnumMap;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.URIResolver;
import javax.xml.transform.sax.SAXResult;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import net.sf.saxon.Controller;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xml.sax.XMLReader;
import org.xml.sax.ContentHandler;
import org.xml.sax.ext.LexicalHandler;
import org.xml.sax.helpers.XMLFilterImpl;

/**
 *
 * @author michael
 */
public class TransformingXMLFilter extends JoiningXMLFilter {

    private SplittingXMLFilter splitter;
    private SplittingXMLFilter privateSplitter;
    private File stylesheet;
    private InputStream stylesheetStream;
    private Templates stylesheetTemplates;
    public static final String TRANSFORMER_FACTORY_CLASS_NAME = "net.sf.saxon.TransformerFactoryImpl";
    private static final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactory.newInstance(TRANSFORMER_FACTORY_CLASS_NAME, null);
    private static final String USAGE = "USAGE: command inputFile stylesheet outputFile";
    private static final Logger logger = Logger.getLogger(TransformingXMLFilter.class);

    private static void printUsage(String errorMessage, int exitCode) {
        System.err.println(errorMessage);
        System.err.println(USAGE);
        System.exit(exitCode);
    }

    public static void main(String[] args) throws SAXException, ParserConfigurationException, TransformerConfigurationException, TransformerException, IOException {
        BasicConfigurator.configure();
        if (args.length != 3) {
            printUsage("wrong number of arguments: "+args.length, -1);
        }

        File inputFile = new File(args[0]);
        File stylesheet = new File(args[1]);
        File outputFile = new File(args[2]);

        TransformingXMLFilter txf = new TransformingXMLFilter();
        try {
            txf.setStylesheet(stylesheet);
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        InputSource inputSource = null;
        try {
            inputSource = new InputSource(new BufferedInputStream(new FileInputStream(inputFile)));
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        BufferedOutputStream bos = null;
        try {
            bos = new BufferedOutputStream(new FileOutputStream(outputFile));
        } catch (FileNotFoundException ex) {
            printUsage(ex.getMessage(), -1);
        }

        Transformer t = stf.newTransformer();
        txf.configureOutputTransformer((Controller) t);

        long start = System.currentTimeMillis();
        t.transform(new SAXSource(txf, inputSource), new StreamResult(bos));
        bos.close();
        System.out.println("transformation complete; duration: " + (System.currentTimeMillis() - start)+ " ms");
    }

    public TransformingXMLFilter() throws ParserConfigurationException, SAXException {
        privateSplitter = new SplittingXMLFilter();
        super.setParent(privateSplitter);
        setUpstreamSplittingFilter(privateSplitter);

        SAXParserFactory spf = SAXParserFactory.newInstance();
        spf.setNamespaceAware(true);
        XMLReader reader = spf.newSAXParser().getXMLReader();
        privateSplitter.setParent(reader);
    }

    @Override
    public void setParent(XMLReader parent) {
        super.setParent(parent);
        privateSplitter = null;
    }

    public void setStreamingParent(XMLReader parent) {
        if (privateSplitter == null) {
            privateSplitter = new SplittingXMLFilter();
            super.setParent(privateSplitter);
            setUpstreamSplittingFilter(privateSplitter);
        }
        privateSplitter.setParent(parent);
    }

    public XMLReader getStreamingParent() {
        if (privateSplitter == null) {
            return null;
        } else {
            return privateSplitter.getParent();
        }
    }

    public void setStylesheetStream(InputStream stylesheetStream) throws TransformerConfigurationException, FileNotFoundException {
        if (stylesheetStream == null) {
            stylesheetTemplates = null;
        } else {
            stylesheetTemplates = stf.newTemplates(new StreamSource(stylesheetStream));
        }
        this.stylesheetStream = stylesheetStream;
    }

    public InputStream getStylesheetStream() {
        return stylesheetStream;
    }

    public void setStylesheet(File stylesheet) throws TransformerConfigurationException, FileNotFoundException {
        if (stylesheet == null) {
            stylesheetTemplates = null;
        } else {
            stylesheetTemplates = stf.newTemplates(new StreamSource(stylesheet));
        }
        this.stylesheet = stylesheet;
    }

    public File getStylesheet() {
        return stylesheet;
    }

    public void configureOutputTransformer(Controller out) {
        if (stylesheetTemplates != null) {
            try {
                Controller c;
                c = (Controller) stylesheetTemplates.newTransformer();
                out.getExecutable().setCharacterMapIndex(c.getExecutable().getCharacterMapIndex());
                out.setOutputProperties(c.getOutputProperties());

            } catch (TransformerConfigurationException ex) {
                throw new RuntimeException(ex);
            }
        }
    }

    public final void setUpstreamSplittingFilter(SplittingXMLFilter splitter) {
        this.splitter = splitter;
{'fraction_non_alphanumeric': 0.08753107418552925, 'fraction_numerical': 0.0030092895459897947, 'mean_word_length': 3.5578413834227787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 58}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1922456', 'n_tokens_mistral': 4148, 'n_tokens_neox': 4050, 'n_words': 930}