<commit_before>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.htmlcleaner;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Comment;
import org.w3c.dom.DOMImplementation;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentType;
import org.w3c.dom.Element;

/**
 * Generate a W3C Document from a SF's HTML Cleaner TagNode.
 *
 * Some code has been copy-pasted from SF's HTML Cleaner code (which is under a BDS license, see
 * http://htmlcleaner.sourceforge.net/license.php). Our goal is to remove this class completely if we can get SF's HTML
 * Cleaner to support the CDATA-related use cases that force us to have this class.
 *
 * Remove when the following issues have been fixed:
 * <ul>
 *     <li>https://sourceforge.net/p/htmlcleaner/bugs/169/</li>
 * </ul>
 *
 * Note: Even though in a public package this code is not meant to be a public API. We've had to put in under the {@code
 * org.htmlcleaner} package because of https://sourceforge.net/p/htmlcleaner/bugs/167/.
 *
 * @version $Id$
 * @since 1.8.2
 */
public class XWikiDOMSerializer
{
    /**
     * The Regex Pattern to recognize a CDATA block.
     */
    private static final Pattern CDATA_PATTERN = Pattern.compile("<!\\[CDATA\\[.*(\\]\\]>|<!\\[CDATA\\[)",
        Pattern.DOTALL);

    private static final String CSS_COMMENT_START = "/*";

    private static final String CSS_COMMENT_END = "*/";

    private static final String JS_COMMENT = "//";

    private static final String NEW_LINE = "\n";

    private static final String SCRIPT_TAG_NAME = "script";

    private static final String STYLE_TAG_NAME = "style";

    private static final String HTML_TAG_NAME = "html";

    /**
     * The HTML Cleaner properties set by the user to control the HTML cleaning.
     */
    private CleanerProperties props;

    /**
     * Whether XML entities should be escaped or not.
     */
    private boolean escapeXml;

    /**
     * @param props the HTML Cleaner properties set by the user to control the HTML cleaning.
     * @param escapeXml if true then escape XML entities
     */
    public XWikiDOMSerializer(CleanerProperties props, boolean escapeXml)
    {
        this.props = props;
        this.escapeXml = escapeXml;
    }

    /**
     * @param documentDocumentBuilder the {@link DocumentBuilder} instance to use, DocumentBuilder is not garantied to
     * be thread safe so at most the safe instance should be used only in the same thread
     * @param rootNode the HTML Cleaner root node to serialize
     * @return the W3C Document object
     * @throws ParserConfigurationException if there's an error during serialization
     */
    public Document createDOM(DocumentBuilder documentDocumentBuilder, TagNode rootNode)
        throws ParserConfigurationException
    {
        DOMImplementation impl = documentDocumentBuilder.getDOMImplementation();

        // Copied from the source code of HTML Cleaner.

        Document document;

        //
        // Where a DOCTYPE is supplied in the input, ensure that this is in the output DOM. See issue #27
        //
        // Note that we may want to fix incorrect DOCTYPEs in future; there are some fairly
        // common patterns for errors with the older HTML4 doctypes.
        //
        if (rootNode.getDocType() != null) {
            String qualifiedName = rootNode.getDocType().getPart1();
            String publicId = rootNode.getDocType().getPublicId();
            String systemId = rootNode.getDocType().getSystemId();

            //
            // If there is no qualified name, set it to html. See bug #153.
            //
            if (qualifiedName == null) {
                qualifiedName = HTML_TAG_NAME;
            }

            DocumentType documentType = impl.createDocumentType(qualifiedName, publicId, systemId);

            //
            // While the qualified name is "HTML" for some DocTypes, we want the actual document root name to be "html".
            // See bug #116
            //
            if (qualifiedName.equals("HTML")) {
                qualifiedName = HTML_TAG_NAME;
            }
            document = impl.createDocument(rootNode.getNamespaceURIOnPath(""), qualifiedName, documentType);
        } else {
            document = documentDocumentBuilder.newDocument();
            Element rootElement = document.createElement(rootNode.getName());
            document.appendChild(rootElement);
        }

        //
        // Copy across root node attributes - see issue 127. Thanks to rasifiel for the patch
        //
        Map<String, String> attributes = rootNode.getAttributes();
        Iterator<Map.Entry<String, String>> entryIterator = attributes.entrySet().iterator();
        while (entryIterator.hasNext()) {
            Map.Entry<String, String> entry = entryIterator.next();
            String attrName = entry.getKey();
            String attrValue = entry.getValue();
            if (escapeXml) {
                attrValue = Utils.escapeXml(attrValue, props, true);
            }

            document.getDocumentElement().setAttribute(attrName, attrValue);

            //
            // Flag the attribute as an ID attribute if appropriate. Thanks to Chris173<commit_msg>[Misc] Move the "HTML" string literal on the left side of string comparison<commit_after>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.htmlcleaner;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Comment;
import org.w3c.dom.DOMImplementation;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentType;
import org.w3c.dom.Element;

/**
 * Generate a W3C Document from a SF's HTML Cleaner TagNode.
 *
 * Some code has been copy-pasted from SF's HTML Cleaner code (which is under a BDS license, see
 * http://htmlcleaner.sourceforge.net/license.php). Our goal is to remove this class completely if we can get SF's HTML
 * Cleaner to support the CDATA-related use cases that force us to have this class.
 *
 * Remove when the following issues have been fixed:
 * <ul>
 *     <li>https://sourceforge.net/p/htmlcleaner/bugs/169/</li>
 * </ul>
 *
 * Note: Even though in a public package this code is not meant to be a public API. We've had to put in under the {@code
 * org.htmlcleaner} package because of https://sourceforge.net/p/htmlcleaner/bugs/167/.
 *
 * @version $Id$
 * @since 1.8.2
 */
public class XWikiDOMSerializer
{
    /**
     * The Regex Pattern to recognize a CDATA block.
     */
    private static final Pattern CDATA_PATTERN = Pattern.compile("<!\\[CDATA\\[.*(\\]\\]>|<!\\[CDATA\\[)",
        Pattern.DOTALL);

    private static final String CSS_COMMENT_START = "/*";

    private static final String CSS_COMMENT_END = "*/";

    private static final String JS_COMMENT = "//";

    private static final String NEW_LINE = "\n";

    private static final String SCRIPT_TAG_NAME = "script";

    private static final String STYLE_TAG_NAME = "style";

    private static final String HTML_TAG_NAME = "html";

    /**
     * The HTML Cleaner properties set by the user to control the HTML cleaning.
     */
    private CleanerProperties props;

    /**
     * Whether XML entities should be escaped or not.
     */
    private boolean escapeXml;

    /**
     * @param props the HTML Cleaner properties set by the user to control the HTML cleaning.
     * @param escapeXml if true then escape XML entities
     */
    public XWikiDOMSerializer(CleanerProperties props, boolean escapeXml)
    {
        this.props = props;
        this.escapeXml = escapeXml;
    }

    /**
     * @param documentDocumentBuilder the {@link DocumentBuilder} instance to use, DocumentBuilder is not garantied to
     * be thread safe so at most the safe instance should be used only in the same thread
     * @param rootNode the HTML Cleaner root node to serialize
     * @return the W3C Document object
     * @throws ParserConfigurationException if there's an error during serialization
     */
    public Document createDOM(DocumentBuilder documentDocumentBuilder, TagNode rootNode)
        throws ParserConfigurationException
    {
        DOMImplementation impl = documentDocumentBuilder.getDOMImplementation();

        // Copied from the source code of HTML Cleaner.

        Document document;

        //
        // Where a DOCTYPE is supplied in the input, ensure that this is in the output DOM. See issue #27
        //
        // Note that we may want to fix incorrect DOCTYPEs in future; there are some fairly
        // common patterns for errors with the older HTML4 doctypes.
        //
        if (rootNode.getDocType() != null) {
            String qualifiedName = rootNode.getDocType().getPart1();
            String publicId = rootNode.getDocType().getPublicId();
            String systemId = rootNode.getDocType().getSystemId();

            //
            // If there is no qualified name, set it to html. See bug #153.
            //
            if (qualifiedName == null) {
                qualifiedName = HTML_TAG_NAME;
            }

            DocumentType documentType = impl.createDocumentType(qualifiedName, publicId, systemId);

            //
            // While the qualified name is "HTML" for some DocTypes, we want the actual document root name to be "html".
            // See bug #116
            //
            if ("HTML".equals(qualifiedName)) {
                qualifiedName = HTML_TAG_NAME;
            }
            document = impl.createDocument(rootNode.getNamespaceURIOnPath(""), qualifiedName, documentType);
        } else {
            document = documentDocumentBuilder.newDocument();
            Element rootElement = document.createElement(rootNode.getName());
            document.appendChild(rootElement);
        }

        //
        // Copy across root node attributes - see issue 127. Thanks to rasifiel for the patch
        //
        Map<String, String> attributes = rootNode.getAttributes();
        Iterator<Map.Entry<String, String>> entryIterator = attributes.entrySet().iterator();
        while (entryIterator.hasNext()) {
            Map.Entry<String, String> entry = entryIterator.next();
            String attrName = entry.getKey();
            String attrValue = entry.getValue();
            if (escapeXml) {
                attrValue = Utils.escapeXml(attrValue, props, true);
            }

            document.getDocumentElement().setAttribute(attrName, attrValue);

            //
            // Flag the attribute as an ID attribute if appropriate. Thanks to Chris173
{'fraction_non_alphanumeric': 0.0870724819579542, 'fraction_numerical': 0.007373705679322246, 'mean_word_length': 2.7007256894049347, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1699441', 'n_tokens_mistral': 3404, 'n_tokens_neox': 3260, 'n_words': 1261}