<commit_before>package gov.nasa.jpl.mbee.mdk.docgen;

import com.nomagic.magicdraw.core.Application;
import com.nomagic.magicdraw.core.GUILog;
import com.nomagic.magicdraw.export.image.ImageExporter;
import com.nomagic.magicdraw.properties.BooleanProperty;
import com.nomagic.magicdraw.properties.ElementProperty;
import com.nomagic.magicdraw.properties.NumberProperty;
import com.nomagic.magicdraw.properties.StringProperty;
import com.nomagic.magicdraw.uml.symbols.DiagramPresentationElement;
import com.nomagic.uml2.ext.magicdraw.classes.mdkernel.*;
import gov.nasa.jpl.mbee.mdk.docgen.docbook.DocumentElement;
import gov.nasa.jpl.mbee.mdk.docgen.view.ViewElement;
import org.apache.commons.lang3.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.*;

public class DocGenUtils {

    /**
     * this was trying to use regex only to convert html tags to docbook tags,
     * should probably switch to just using jsoup
     */
    @SuppressWarnings("serial")
    public static final Map<String, String> html2docbookConvert = new HashMap<String, String>() {
        {
            put("<p>|<p [^>]*>", "<para>");
            put("</p>", "</para>");
            put("<ul>|<ul [^>]*>",
                    "<itemizedlist spacing=\"compact\">");
            put("</ul>", "</itemizedlist>");
            put("<ol>|<ol [^>]*>",
                    "<orderedlist spacing=\"compact\">");
            put("</ol>", "</orderedlist>");
            put("<li>|<li [^>]*>",
                    "<listitem><para>");
            put("</li>", "</para></listitem>");
            put("<b>|<b [^>]*>|<em>|<em [^>]*>|<strong>|<strong [^>]*>",
                    "<emphasis role=\"bold\">");
            put("<h[1-6]>|<h[1-6] [^>]*>", "<para><emphasis role=\"bold\">");
            put("<s>|<strike>|<s [^>]*>|<strike [^>]*>",
                    "<emphasis role=\"strikethrough\">");
            put("<i>|<i [^>]*>", "<emphasis>");
            put("<u>|<u [^>]*>",
                    "<emphasis role=\"underline\">");
            put("<span>|<span [^>]*>|</span>|<br>|<br/>|</br>|<br />",
                    "");
            put("</b>|</i>|</u>|</strong>|</em>|</s>|</strike>",
                    "</emphasis>");
            put("</h[1-6]>", "</emphasis></para>");
            put("<font [^>]*>|</font>", "");
            put("<sup>|<sup [^>]*>",
                    "<superscript>");
            put("<sub>|<sub [^>]*>",
                    "<subscript>");
            put("</sup>", "</superscript>");
            put("</sub>", "</subscript>");
            put("<a href=\"(http[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"(file[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"(mailto[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"mdel://([^\"&^\\?]+)(\\?[^\"]*)?\">([^<]*)</a>",
                    "<link linkend=\"$1\">$3</link>");
            put("<img [^>]*src=\"([^>]+)\"[^>]*></img>", "<imageobject><imagedata fileref=\"$1\" scalefit=\"1\"/></imageobject>");
            put("<img [^>]*src=\"([^>]+)\"[^>]*/>", "<imageobject><imagedata fileref=\"$1\" scalefit=\"1\"/></imageobject>");
            put("<pre>|<pre [^>]*>", "<screen>");
            put("</pre>", "</screen>");
            put("<svg",
                    "<mediaobject><imageobject><imagedata><svg");
            put("</svg>",
                    "</svg></imagedata></imageobject></mediaobject>");
            put("&nbsp;", "&#160;");
            put("&sup2;",
                    "<superscript>2</superscript>");
            put("&sup3;",
                    "<superscript>3</superscript>");
        }
    };

    /**
     * docbook ignores regular white space in table cells, this is to force
     * indentation in docbook, 1 indent is 4 spaces
     *
     * @param name
     * @param depth
     * @return
     */
    public static String getIndented(String name, int depth) {
        String space = "";
        for (int i = 1; i < depth; i++) {
            space += "&#xA0;&#xA0;&#xA0;&#xA0;";
        }
        return space + name;
    }

    /**
     * given any object tries to return a string representation suitable for use
     * in docbook<br/>
     *
     * @param s
     * @return
     */
    public static String fixString(Object s) {
        return fixString(s, true);
    }

    public static String fixString(Object s, boolean convertHtml) {
        String rv;
        // may want to look at
        // com.nomagic.magicdraw.uml.RepresentationTextCreator.getRepresentedText
        if (s instanceof String) {
            if (((String) s).contains("<html>")) {
                if (convertHtml) {
                    return StringEscapeUtils.escapeHtml4((String) s);
                }
                else {
                    return gov.nasa.jpl.mbee.mdk.util.Utils.stripHtmlWrapper((String) s);
                }
            }
            else {
                return StringEscapeUtils.escapeHtml4(((String) s)
                        .replaceAll("&(?![A-Za-z#0-9]+;)", "&amp;").replaceAll("<([>=\\s])", "&lt;$1")
                        .replaceAll("<<", "&lt;&lt;").replaceAll("<(?![^>]+>)", "&lt;"));
            }
        }
        else if (s instanceof Integer) {
            return Integer.toString((Integer) s);
        }
        else if (s instanceof InstanceValue) {
            InstanceSpecification is = ((InstanceValue) s).getInstance();
            if (is != null) {
                return fixString(is.getName());
            }
        }<commit_msg>Replaced HTML escaping logic with HTML to XML entities to capture original intent.<commit_after>package gov.nasa.jpl.mbee.mdk.docgen;

import com.nomagic.magicdraw.core.Application;
import com.nomagic.magicdraw.core.GUILog;
import com.nomagic.magicdraw.export.image.ImageExporter;
import com.nomagic.magicdraw.properties.BooleanProperty;
import com.nomagic.magicdraw.properties.ElementProperty;
import com.nomagic.magicdraw.properties.NumberProperty;
import com.nomagic.magicdraw.properties.StringProperty;
import com.nomagic.magicdraw.uml.symbols.DiagramPresentationElement;
import com.nomagic.uml2.ext.magicdraw.classes.mdkernel.*;
import gov.nasa.jpl.mbee.mdk.docgen.docbook.DocumentElement;
import gov.nasa.jpl.mbee.mdk.docgen.view.ViewElement;
import org.apache.commons.lang3.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class DocGenUtils {

    /**
     * this was trying to use regex only to convert html tags to docbook tags,
     * should probably switch to just using jsoup
     */
    @SuppressWarnings("serial")
    public static final Map<String, String> html2docbookConvert = new HashMap<String, String>() {
        {
            put("<p>|<p [^>]*>", "<para>");
            put("</p>", "</para>");
            put("<ul>|<ul [^>]*>",
                    "<itemizedlist spacing=\"compact\">");
            put("</ul>", "</itemizedlist>");
            put("<ol>|<ol [^>]*>",
                    "<orderedlist spacing=\"compact\">");
            put("</ol>", "</orderedlist>");
            put("<li>|<li [^>]*>",
                    "<listitem><para>");
            put("</li>", "</para></listitem>");
            put("<b>|<b [^>]*>|<em>|<em [^>]*>|<strong>|<strong [^>]*>",
                    "<emphasis role=\"bold\">");
            put("<h[1-6]>|<h[1-6] [^>]*>", "<para><emphasis role=\"bold\">");
            put("<s>|<strike>|<s [^>]*>|<strike [^>]*>",
                    "<emphasis role=\"strikethrough\">");
            put("<i>|<i [^>]*>", "<emphasis>");
            put("<u>|<u [^>]*>",
                    "<emphasis role=\"underline\">");
            put("<span>|<span [^>]*>|</span>|<br>|<br/>|</br>|<br />",
                    "");
            put("</b>|</i>|</u>|</strong>|</em>|</s>|</strike>",
                    "</emphasis>");
            put("</h[1-6]>", "</emphasis></para>");
            put("<font [^>]*>|</font>", "");
            put("<sup>|<sup [^>]*>",
                    "<superscript>");
            put("<sub>|<sub [^>]*>",
                    "<subscript>");
            put("</sup>", "</superscript>");
            put("</sub>", "</subscript>");
            put("<a href=\"(http[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"(file[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"(mailto[^\"]+)\">([^<]*)</a>",
                    "<link xlink:href=\"$1\">$2</link>");
            put("<a href=\"mdel://([^\"&^\\?]+)(\\?[^\"]*)?\">([^<]*)</a>",
                    "<link linkend=\"$1\">$3</link>");
            put("<img [^>]*src=\"([^>]+)\"[^>]*></img>", "<imageobject><imagedata fileref=\"$1\" scalefit=\"1\"/></imageobject>");
            put("<img [^>]*src=\"([^>]+)\"[^>]*/>", "<imageobject><imagedata fileref=\"$1\" scalefit=\"1\"/></imageobject>");
            put("<pre>|<pre [^>]*>", "<screen>");
            put("</pre>", "</screen>");
            put("<svg",
                    "<mediaobject><imageobject><imagedata><svg");
            put("</svg>",
                    "</svg></imagedata></imageobject></mediaobject>");
            put("&nbsp;", "&#160;");
            put("&sup2;",
                    "<superscript>2</superscript>");
            put("&sup3;",
                    "<superscript>3</superscript>");
        }
    };
    private static final Pattern ENTITY_PATTERN = Pattern.compile("(&[^\\s]+?;)");

    /**
     * docbook ignores regular white space in table cells, this is to force
     * indentation in docbook, 1 indent is 4 spaces
     *
     * @param name
     * @param depth
     * @return
     */
    public static String getIndented(String name, int depth) {
        String space = "";
        for (int i = 1; i < depth; i++) {
            space += "&#xA0;&#xA0;&#xA0;&#xA0;";
        }
        return space + name;
    }

    /**
     * Convert a String of HTML with named HTML entities to the
     * same String with entities converted to numbered XML entities
     *
     * @param html
     * @return xml
     */
    public static String htmlToXmlEntities(String html) {
        StringBuffer stringBuffer = new StringBuffer();
        Matcher matcher = ENTITY_PATTERN.matcher(html);

        while (matcher.find()) {
            String replacement = htmlEntityToXmlEntity(matcher.group(1));
            matcher.appendReplacement(stringBuffer, "");
            stringBuffer.append(replacement);
        }

        matcher.appendTail(stringBuffer);
        return stringBuffer.toString();
    }

    /**
     * Replace an HTML entity with an XML entity
     *
     * @param html
     * @return xml
     */
    private static String htmlEntityToXmlEntity(String html) {
        return StringEscapeUtils.escapeXml11(StringEscapeUtils.unescapeHtml4(html));
    }

    /**
     * given any object tries to return a string representation suitable for use
     * in docbook<br/>
     *
     * @param s
     * @return
     */
    public static String fixString(Object s) {
        return fixString(s, true);
    }

    public static String fixString(Object s, boolean convertHtml) {
        String rv;
        // may want to look at
        // com.nomagic.magicdraw.uml.RepresentationTextCreator.getRepresentedText
        if (s instanceof String) {
            if (((String) s).contains("<html>")) {
                if (convertHtml) {
                    return htmlToXmlEntities((String) s);
                }
                else {
                    return gov.nasa.jpl.mbee.mdk.util.Utils.stripHtmlWrapper((String) s);
                }
            }
            else {
                return htmlToXmlEntities(((String) s)
                        .replaceAll("&(?![A-Za-z#0-9]+;)", "&amp;").replaceAll("<([>=\\s])", "&lt;$1")
                        .replaceAll("<<", "&lt;&lt;").replaceAll("<(?![^>]+>)", "&lt;"));
            }
        }
        else if (s instanceof Integer) {
            return Integer.toString((Integer) s);
        }
        else if (s instanceof InstanceValue) {
            InstanceSpecification is = ((InstanceValue) s).getInstance();
            if (is != null) {
                return fixString(is.getName());
            }
        }
{'fraction_non_alphanumeric': 0.2159859883767216, 'fraction_numerical': 0.006528142663800653, 'mean_word_length': 2.0460717749757515, 'pattern_counts': {'":': 0, '<': 265, '<?xml version=': 0, '>': 295, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2012377', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3817, 'n_words': 677}