<commit_before>package com.occamlab.te.parsers;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedOutputStream;
import java.io.PipedInputStream;
import java.io.PrintWriter;

import java.lang.ClassLoader;
import java.lang.Thread;

import java.net.URL;
import java.net.URLConnection;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.NodeList;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.thaiopensource.util.PropertyMapBuilder;
import com.thaiopensource.validate.SchemaReader;
import com.thaiopensource.validate.SchemaReaderLoader;
import com.thaiopensource.validate.ValidationDriver;
import com.thaiopensource.validate.schematron.SchematronProperty;
import com.thaiopensource.validate.ValidateProperty;

import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.dom.DOMSource;

import com.occamlab.te.ErrorHandlerImpl;

/**
 * Validates the given XML resource against the rules specified in a Schematron
 * (v1.5) file. Used in conjunction with standard XML Schema validator to
 * provide more thorough validation coverage.
 *
 * @author jparrpearson
 */
public class SchematronValidatingParser {

    /** Configuration info for Schematron validation. */
    private PropertyMapBuilder propMapBuilder = null;

    /** Schema reader for Schematron validation. */
    private SchemaReader schematronReader = null;

    /** Namespace URI for the Schematron assertion language (v 1.5). */
    public static final String SCHEMATRON_NS_URI = "http://www.ascc.net/xml/schematron";

    /**
     * Creates and initializes the schematron reader used in the validation
     * process.
     */
    private void initSchematronReader() {
        this.propMapBuilder = new PropertyMapBuilder();
        // set general properties for schematron validation
        SchematronProperty.DIAGNOSE.add(this.propMapBuilder);
        SchemaReaderLoader loader = new SchemaReaderLoader();
        this.schematronReader = loader.createSchemaReader(SCHEMATRON_NS_URI);
        assert null != this.schematronReader : "Unable to create a SchemaReader for this schema language: "
                + SCHEMATRON_NS_URI;
    }

    /**
     * Converts an org.w3c.dom.Document element to an java.io.InputStream.
     *
     * @param edoc
     *            The org.w3c.dom.Document to be converted
     * @return The InputStream value of the passed doument
     */
    public InputStream DocumentToInputStream(org.w3c.dom.Document edoc)
            throws IOException {

        // Create the input and output for use in the transformation
        final org.w3c.dom.Document doc = edoc;
        final PipedOutputStream pos = new PipedOutputStream();
        PipedInputStream pis = new PipedInputStream();
        pis.connect(pos);

        (new Thread(new Runnable() {

            public void run() {
                // Use the Transformer.transform() method to save the Document
                // to a StreamResult
                try {
                    TransformerFactory tFactory = TransformerFactory
                            .newInstance();
                    Transformer transformer = tFactory.newTransformer();
                    transformer.setOutputProperty("encoding", "ISO-8859-1");
                    transformer.setOutputProperty("indent", "yes");
                    transformer.transform(new DOMSource(doc), new StreamResult(
                            pos));
                } catch (Exception _ex) {
                    throw new RuntimeException(
                            "Failed to tranform org.w3c.dom.Document to PipedOutputStream",
                            _ex);
                } finally {
                    try {
                        pos.close();
                    } catch (IOException e) {

                    }
                }
            }
        }, "MyClassName.convert(org.w3c.dom.Document edoc)")).start();

        return pis;
    }

    /**
     * Checks the given schematron phase for the XML file and returns the
     * validation status.
     *
     * @param doc
     *            The XML file to validate (Document)
     * @param schemaFile
     *            The string path to the schematron file to use
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return Whether there were validation errors or not (boolean)
     */
    public boolean checkSchematronRules(Document doc, String schemaFile,
            String phase) {

        // The validation state (true = no validation errors)
        boolean isValid = false;

	if (doc == null || doc.getDocumentElement() == null) return isValid;

        // Convert doc to InputSource
        InputSource xmlInputSource = null;
        try {
            InputStream inputStream = DocumentToInputStream(doc);
            xmlInputSource = new InputSource(inputStream);
        } catch (IOException e) {
            e.printStackTrace();
        }

        // Load schematron file
        File f = null;
        InputSource schema = null;
        FileInputStream fileInputStream = null;
        try {
            // Use ClassLoader to load schematron off classpath
            ClassLoader loader = Thread.currentThread().getContextClassLoader();
            URL url = loader.getResource(schemaFile);
            f = new File(url.getFile());
            // f = new File(schemaFile);
            fileInputStream = new FileInputStream(f);
            schema = new InputSource(fileInputStream);
        } catch (FileNotFoundException e) {
            assert false : "Entity body not found. " + e.toString();
        }
        ValidationDriver driver = createSchematronDriver(phase);
        assert null != driver : "Unable to create Schematron ValidationDriver";
        try {
            // Validate using the thaiopensource validation method
            if (driver.loadSchema(schema)) {
                isValid = driver.validate(xmlInputSource);
            } else {
                assert false : ("Failed to load Schematron schema: "
                        + schemaFile + "\nIs the schema valid? Is the phase defined?");
            }
        } catch (SAXException e) {
            assert false : e.toString();
        } catch (IOException e) {
            assert false : e.toString();
        } finally {
            try {
                fileInputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return isValid;
    }

    /**
     * Checks the given schematron phase for the XML file and returns the
     * validation status (takes schematron file, not string location). New and
     * ADVANCED! (team engine can't work with overloaded methods :P)
     *
     * @param doc
     *            The XML file to validate (Document)
     * @param schemaFile
     *            The file object of the schematron file to validate with
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return Whether there were validation errors or not (boolean)
     */
    public boolean checkSchematronRulesAdv(InputSource inputDoc,
            File schemaFile, String phase) {

        // The validation state (true = no validation errors)
        boolean isValid = false;

        // Load schematron file
        ValidationDriver driver = createSchematronDriver(phase);
        assert null != driver : "Unable to create Schematron ValidationDriver";
        InputSource is = null;
        try {
            FileInputStream fis = new FileInputStream(schemaFile);
            is = new InputSource(fis);
        } catch (Exception e) {
            e.printStackTrace();
        }

        try {
            // Validate using the thaiopensource validation method
            if (driver.loadSchema(is)) {
                isValid = driver.validate(inputDoc);
            } else {
                assert false : ("Failed to load Schematron schema: "
                        + schemaFile + "\nIs the schema valid? Is the phase defined?");
            }
        } catch (SAXException e) {
            assert false : e.toString();
        } catch (IOException e) {
            assert false : e.toString();
        }

        return isValid;
    }

    /**
     * Sets up the schematron reader with all the necessary parameters. Calls
     * initSchematronReader() to do further setup of the validation driver.
     *
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return The ValidationDriver to use in validating the XML document
     */
    private ValidationDriver createSchematronDriver(String phase) {
        initSchematronReader();

        PropertyMapBuilder schematronConfig = this.propMapBuilder;
	// add error handler
        eh = new ErrorHandlerImpl("Schematron", outputLogger);
        ValidateProperty.ERROR_HANDLER.put(schematronConfig, eh);
        // validate using default phase
        SchematronProperty.PHASE.put(schematronConfig, "Default");
        if (null != phase && phase.length() > 0) {
            SchematronProperty.PHASE.put(schematronConfig, phase);
        }
        SchemaReader reader = this.schematronReader;
        ValidationDriver validator = new ValidationDriver(schematronConfig
                .toPropertyMap(), reader);
        return validator;
    }

    private static String schemaLocation = null;

    private static String phase = null;

    private static String type = null;

    private static ErrorHandlerImpl eh = null;
<commit_msg>Fixed method for using schematron parser outside of request element<commit_after>package com.occamlab.te.parsers;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedOutputStream;
import java.io.PipedInputStream;
import java.io.PrintWriter;

import java.lang.ClassLoader;
import java.lang.Thread;

import java.net.URL;
import java.net.URLConnection;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.NodeList;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.thaiopensource.util.PropertyMapBuilder;
import com.thaiopensource.validate.SchemaReader;
import com.thaiopensource.validate.SchemaReaderLoader;
import com.thaiopensource.validate.ValidationDriver;
import com.thaiopensource.validate.schematron.SchematronProperty;
import com.thaiopensource.validate.ValidateProperty;

import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.dom.DOMSource;

import com.occamlab.te.ErrorHandlerImpl;

/**
 * Validates the given XML resource against the rules specified in a Schematron
 * (v1.5) file. Used in conjunction with standard XML Schema validator to
 * provide more thorough validation coverage.
 *
 * @author jparrpearson
 */
public class SchematronValidatingParser {

    /** Configuration info for Schematron validation. */
    private PropertyMapBuilder propMapBuilder = null;

    /** Schema reader for Schematron validation. */
    private SchemaReader schematronReader = null;

    /** Namespace URI for the Schematron assertion language (v 1.5). */
    public static final String SCHEMATRON_NS_URI = "http://www.ascc.net/xml/schematron";

    /**
     * Creates and initializes the schematron reader used in the validation
     * process.
     */
    private void initSchematronReader() {
        this.propMapBuilder = new PropertyMapBuilder();
        // set general properties for schematron validation
        SchematronProperty.DIAGNOSE.add(this.propMapBuilder);
        SchemaReaderLoader loader = new SchemaReaderLoader();
        this.schematronReader = loader.createSchemaReader(SCHEMATRON_NS_URI);
        assert null != this.schematronReader : "Unable to create a SchemaReader for this schema language: "
                + SCHEMATRON_NS_URI;
    }

    /**
     * Converts an org.w3c.dom.Document element to an java.io.InputStream.
     *
     * @param edoc
     *            The org.w3c.dom.Document to be converted
     * @return The InputStream value of the passed doument
     */
    public InputStream DocumentToInputStream(org.w3c.dom.Document edoc)
            throws IOException {

        // Create the input and output for use in the transformation
        final org.w3c.dom.Document doc = edoc;
        final PipedOutputStream pos = new PipedOutputStream();
        PipedInputStream pis = new PipedInputStream();
        pis.connect(pos);

        (new Thread(new Runnable() {

            public void run() {
                // Use the Transformer.transform() method to save the Document
                // to a StreamResult
                try {
                    TransformerFactory tFactory = TransformerFactory
                            .newInstance();
                    Transformer transformer = tFactory.newTransformer();
                    transformer.setOutputProperty("encoding", "ISO-8859-1");
                    transformer.setOutputProperty("indent", "yes");
                    transformer.transform(new DOMSource(doc), new StreamResult(
                            pos));
                } catch (Exception _ex) {
                    throw new RuntimeException(
                            "Failed to tranform org.w3c.dom.Document to PipedOutputStream",
                            _ex);
                } finally {
                    try {
                        pos.close();
                    } catch (IOException e) {

                    }
                }
            }
        }, "MyClassName.convert(org.w3c.dom.Document edoc)")).start();

        return pis;
    }

    /**
     * Checks the given schematron phase for the XML file and returns the
     * validation status.
     *
     * @param doc
     *            The XML file to validate (Document)
     * @param schemaFile
     *            The string path to the schematron file to use
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return Whether there were validation errors or not (boolean)
     */
    public boolean checkSchematronRules(Document doc, String schemaFile,
            String phase) {

        // The validation state (true = no validation errors)
        boolean isValid = false;

	if (doc == null || doc.getDocumentElement() == null) return isValid;

        // Convert doc to InputSource
        InputSource xmlInputSource = null;
        try {
            InputStream inputStream = DocumentToInputStream(doc);
            xmlInputSource = new InputSource(inputStream);
        } catch (IOException e) {
            e.printStackTrace();
        }

        // Load schematron file
        File f = null;
        InputSource schema = null;
        FileInputStream fileInputStream = null;
        try {
            // Use ClassLoader to load schematron off classpath
            ClassLoader loader = Thread.currentThread().getContextClassLoader();
            URL url = loader.getResource(schemaFile);
            f = new File(url.getFile());
            // f = new File(schemaFile);
            fileInputStream = new FileInputStream(f);
            schema = new InputSource(fileInputStream);
        } catch (FileNotFoundException e) {
            assert false : "Entity body not found. " + e.toString();
        }
        ValidationDriver driver = createSchematronDriver(phase);
        assert null != driver : "Unable to create Schematron ValidationDriver";
        try {
            // Validate using the thaiopensource validation method
            if (driver.loadSchema(schema)) {
                isValid = driver.validate(xmlInputSource);
            } else {
                assert false : ("Failed to load Schematron schema: "
                        + schemaFile + "\nIs the schema valid? Is the phase defined?");
            }
        } catch (SAXException e) {
            assert false : e.toString();
        } catch (IOException e) {
            assert false : e.toString();
        } finally {
            try {
                fileInputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return isValid;
    }

    /**
     * Checks the given schematron phase for the XML file and returns the
     * validation status (takes schematron file, not string location). New and
     * ADVANCED! (team engine can't work with overloaded methods :P)
     *
     * @param doc
     *            The XML file to validate (Document)
     * @param schemaFile
     *            The file object of the schematron file to validate with
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return Whether there were validation errors or not (boolean)
     */
    public boolean checkSchematronRulesAdv(InputSource inputDoc,
            File schemaFile, String phase) {

        // The validation state (true = no validation errors)
        boolean isValid = false;

        // Load schematron file
        ValidationDriver driver = createSchematronDriver(phase);
        assert null != driver : "Unable to create Schematron ValidationDriver";
        InputSource is = null;
        try {
            FileInputStream fis = new FileInputStream(schemaFile);
            is = new InputSource(fis);
        } catch (Exception e) {
            e.printStackTrace();
        }

        try {
            // Validate using the thaiopensource validation method
            if (driver.loadSchema(is)) {
                isValid = driver.validate(inputDoc);
            } else {
                assert false : ("Failed to load Schematron schema: "
                        + schemaFile + "\nIs the schema valid? Is the phase defined?");
            }
        } catch (SAXException e) {
            assert false : e.toString();
        } catch (IOException e) {
            assert false : e.toString();
        }

        return isValid;
    }

    /**
     * Sets up the schematron reader with all the necessary parameters. Calls
     * initSchematronReader() to do further setup of the validation driver.
     *
     * @param phase
     *            The string phase name (contained in schematron file)
     * @return The ValidationDriver to use in validating the XML document
     */
    private ValidationDriver createSchematronDriver(String phase) {
        initSchematronReader();

        PropertyMapBuilder schematronConfig = this.propMapBuilder;
	// add error handler
	if (outputLogger == null) {
		outputLogger = new PrintWriter(System.out);
	}
        eh = new ErrorHandlerImpl("Schematron", outputLogger);
        ValidateProperty.ERROR_HANDLER.put(schematronConfig, eh);
        // validate using default phase
        SchematronProperty.PHASE.put(schematronConfig, "Default");
        if (null != phase && phase.length() > 0) {
            SchematronProperty.PHASE.put(schematronConfig, phase);
        }
        SchemaReader reader = this.schematronReader;
        ValidationDriver validator = new ValidationDriver(schematronConfig
                .toPropertyMap(), reader);
        return validator;
    }

    private static String schemaLocation = null;

    private static String phase = null;

    private static String type = null;

    private static ErrorHandlerImpl eh = null;

{'fraction_non_alphanumeric': 0.08072179539021432, 'fraction_numerical': 0.0019207440355843105, 'mean_word_length': 2.184451955577016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2279316', 'n_tokens_mistral': 5085, 'n_tokens_neox': 4921, 'n_words': 1501}