<commit_before>package org.opens.tanaguru.contentadapter.html;

import java.io.IOException;

import java.util.logging.Level;
import java.util.logging.Logger;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.PrettyXmlSerializer;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XmlSerializer;

import org.opens.tanaguru.contentadapter.HTMLCleaner;

public class HTMLCleanerImpl extends AbstractHTMLCleaner implements HTMLCleaner {
    static final String CORRECTOR_NAME = "HTMLCleaner";

    public HTMLCleanerImpl() {
        super();
    }

    public void run() {
        try {
            HtmlCleaner cleaner = new HtmlCleaner();
            TagNode node = cleaner.clean(dirtyHTML);
            // to avoid SAX Fatal Error related to namespace
            node.removeAttribute("xml:lang");
            node.removeAttribute("xmlns:xml");
            // node.removeAttribute("xml:lang");
            CleanerProperties props = cleaner.getProperties();

            //XXX verify each property 
//            props.setAdvancedXmlEscape(false);
//            props.setAllowHtmlInsideAttributes(true);
//            props.setAllowMultiWordAttributes(true);
//            props.setIgnoreQuestAndExclam(false);
//            props.setNamespacesAware(true);
            props.setOmitComments(false);
            props.setOmitDeprecatedTags(false);
            props.setOmitDoctypeDeclaration(false);
            props.setOmitHtmlEnvelope(false);
            props.setOmitUnknownTags(false);
            props.setOmitXmlDeclaration(false);
//            props.setRecognizeUnicodeChars(true);
//            props.setTranslateSpecialEntities(false);
//            props.setTreatDeprecatedTagsAsContent(false);
//            props.setTreatUnknownTagsAsContent(false);
//            props.setUseCdataForScriptAndStyle(false);
//            props.setUseEmptyElementTags(true);

            XmlSerializer serializer = new PrettyXmlSerializer(props);
            result = serializer.getXmlAsString(node);
        } catch (IOException ex) {
            Logger.getLogger(HTMLCleanerImpl.class.getName()).log(Level.SEVERE,
                    null, ex);
            throw new RuntimeException(ex);
        }
    }

    public String getCorrectorName() {
        return CORRECTOR_NAME;
    }
}
<commit_msg>contentAdapter : modification of the HTML cleaner properties to remove comments in clean-up source and thus avoid "The character sequence "]]>" must not appear in content unless used to mark the end of a CDATA section" error<commit_after>package org.opens.tanaguru.contentadapter.html;

import java.io.IOException;

import java.util.logging.Level;
import java.util.logging.Logger;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.PrettyXmlSerializer;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XmlSerializer;

import org.opens.tanaguru.contentadapter.HTMLCleaner;

public class HTMLCleanerImpl extends AbstractHTMLCleaner implements HTMLCleaner {
    static final String CORRECTOR_NAME = "HTMLCleaner";
    private static final String XML_ATTRIBUTE = "xmlns:xml";
    private static final String LANG_ATTRIBUTE = "xml:lang";

    public HTMLCleanerImpl() {
        super();
    }

    @Override
    public void run() {
        try {
            HtmlCleaner cleaner = new HtmlCleaner();
            CleanerProperties props = cleaner.getProperties();
//            props.setAdvancedXmlEscape(true);
//            props.setAllowHtmlInsideAttributes(false);
//            props.setAllowMultiWordAttributes(true);
//            props.setIgnoreQuestAndExclam(true);
//            props.setNamespacesAware(true);
//            props.setRecognizeUnicodeChars(true);
//            props.setTranslateSpecialEntities(false);
//            props.setTreatDeprecatedTagsAsContent(false);
//            props.setTreatUnknownTagsAsContent(false);
//            props.setUseEmptyElementTags(true);
//            props.setOmitDeprecatedTags(false);
//            props.setOmitHtmlEnvelope(false);
//            props.setOmitUnknownTags(false);
//            props.setOmitXmlDeclaration(false);
            props.setOmitComments(true);
            props.setOmitDoctypeDeclaration(false);
            props.setUseCdataForScriptAndStyle(true);
            // to avoid SAX Fatal Error related to namespace
            TagNode node = cleaner.clean(dirtyHTML);
            node.removeAttribute(LANG_ATTRIBUTE);
            node.removeAttribute(XML_ATTRIBUTE);
            XmlSerializer serializer = new PrettyXmlSerializer(props);
            result = serializer.getAsString(node);
        } catch (IOException ex) {
            Logger.getLogger(HTMLCleanerImpl.class.getName()).log(Level.SEVERE,
                    null, ex);
            throw new RuntimeException(ex);
        }
    }

    @Override
    public String getCorrectorName() {
        return CORRECTOR_NAME;
    }
}

{'fraction_non_alphanumeric': 0.08994501178318931, 'fraction_numerical': 0.0, 'mean_word_length': 2.4835841313269493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7664648', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1344, 'n_words': 245}