<commit_before>/**
 * Copyright (C) 2012-2013 <NAME> (born 1979)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package debiki

import com.debiki.core._
//import com.twitter.ostrich.stats.Stats
import java.{lang => jl, util => ju, io => jio}
import javax.{script => js}
import scala.collection.JavaConversions._
import _root_.scala.xml.{NodeSeq, Node, Elem, Text, XML, Attribute}
import FlagType.FlagType
import Prelude._
import DebikiHttp._
import HtmlUtils._


object HtmlPageSerializer {


  /**
    * @param nofollowArticle If true, links in the page body will be rel=nofollow.
    *   The only time it's set to false, is when we serve data to search engines, but
    *   usually we're replying to ajax requests. So defaults to true (and this feels safe).
    */
  def markupTextOf(post: Post, hostAndPort: String, nofollowArticle: Boolean = true): String =
    _markupTextOf(post, hostAndPort, nofollowArticle).toString


  // COULD move to HtmlPostSerializer.
  def _markupTextOf(post: Post, hostAndPort: String, nofollowArticle: Boolean = true,
        showUnapproved: ShowUnapproved = ShowUnapproved.None): NodeSeq = {

    val isArticle = post.id == PageParts.BodyId

    // Use nofollow links in people's comments, so Google won't punish
    // the website if someone posts spam.
    val makeNofollowLinks = !isArticle || (nofollowArticle && post.id == PageParts.BodyId)

    val text: String =
      if (showUnapproved.shallShow(post)) post.currentText
      else post.approvedText getOrElse {
        return Nil
      }

    markdownToSafeHtml(
      text, hostAndPort, allowClassIdDataAttrs = isArticle, makeNofollowLinks)
  }


  /** Converts text to xml, returns (html, approx-line-count).
   *
   * Splits into <p>:s and <br>:s at newlines, does nothing else.
   */
  def textToHtml(text: String, charsPerLine: Int = 80): (NodeSeq, Int) = {
    var lineCount = 0
    val xml =
      // Two newlines end a paragraph.
      for (para <- text.split("\n\n").toList)
      yield {
        // One newline results in a row break.
        val lines = para.split("\n").zipWithIndex.flatMap(lineAndIndex => {
          lineCount += 1 + lineAndIndex._1.length / charsPerLine
          if (lineAndIndex._2 == 0) Text(lineAndIndex._1)
          else <br/> ++ Text(lineAndIndex._1)
        })
        <p>{lines}</p>
      }
    (xml, lineCount)
  }


  /** The Nashorn Javascript engine isn't thread safe. */
  private val threadLocalJavascriptEngine = new jl.ThreadLocal[js.ScriptEngine]


  private def javascriptEngine: js.ScriptEngine = {
    val engineOrNull = threadLocalJavascriptEngine.get
    if (engineOrNull != null)
      return engineOrNull

    val newEngine = new js.ScriptEngineManager(null).getEngineByName("nashorn")
    // Remarkable seems to expect `global` to exist. (Remarkable is a CommonMark Markdown renderer.)
    newEngine.eval("var global = this;")
    newEngine.eval(new jio.FileReader("public/res/remarkable.min.js"))
    newEngine.eval(new jio.FileReader("public/res/html-sanitizer-bundle.js"))
    newEngine.eval(new jio.FileReader("public/res/mentions-remarkable-plugin.js"))
    newEngine.eval(i"""
      |remarkable = new Remarkable({ html: true });
      |remarkable.use(MentionsRemarkablePlugin());
      |
      |function renderCommonMark(source) {
      |  return remarkable.render(source);
      |}
      |""")
    threadLocalJavascriptEngine.set(newEngine)
    newEngine
  }


  /** Converts CommonMark markdown to HTML.
    */
  def markdownToSafeHtml(source: String, hostAndPort: String,
        allowClassIdDataAttrs: Boolean, makeLinksNofollow: Boolean = true): NodeSeq = {
    val htmlTextUnsafe = javascriptEngine.asInstanceOf[js.Invocable].invokeFunction(
      "renderCommonMark", source).asInstanceOf[String]
<commit_msg>Make file paths work in prod builds.<commit_after>/**
 * Copyright (C) 2012-2013 <NAME> (born 1979)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package debiki

import com.debiki.core._
//import com.twitter.ostrich.stats.Stats
import java.{lang => jl, util => ju, io => jio}
import javax.{script => js}
import scala.collection.JavaConversions._
import _root_.scala.xml.{NodeSeq, Node, Elem, Text, XML, Attribute}
import FlagType.FlagType
import Prelude._
import DebikiHttp._
import HtmlUtils._


object HtmlPageSerializer {


  /**
    * @param nofollowArticle If true, links in the page body will be rel=nofollow.
    *   The only time it's set to false, is when we serve data to search engines, but
    *   usually we're replying to ajax requests. So defaults to true (and this feels safe).
    */
  def markupTextOf(post: Post, hostAndPort: String, nofollowArticle: Boolean = true): String =
    _markupTextOf(post, hostAndPort, nofollowArticle).toString


  // COULD move to HtmlPostSerializer.
  def _markupTextOf(post: Post, hostAndPort: String, nofollowArticle: Boolean = true,
        showUnapproved: ShowUnapproved = ShowUnapproved.None): NodeSeq = {

    val isArticle = post.id == PageParts.BodyId

    // Use nofollow links in people's comments, so Google won't punish
    // the website if someone posts spam.
    val makeNofollowLinks = !isArticle || (nofollowArticle && post.id == PageParts.BodyId)

    val text: String =
      if (showUnapproved.shallShow(post)) post.currentText
      else post.approvedText getOrElse {
        return Nil
      }

    markdownToSafeHtml(
      text, hostAndPort, allowClassIdDataAttrs = isArticle, makeNofollowLinks)
  }


  /** Converts text to xml, returns (html, approx-line-count).
   *
   * Splits into <p>:s and <br>:s at newlines, does nothing else.
   */
  def textToHtml(text: String, charsPerLine: Int = 80): (NodeSeq, Int) = {
    var lineCount = 0
    val xml =
      // Two newlines end a paragraph.
      for (para <- text.split("\n\n").toList)
      yield {
        // One newline results in a row break.
        val lines = para.split("\n").zipWithIndex.flatMap(lineAndIndex => {
          lineCount += 1 + lineAndIndex._1.length / charsPerLine
          if (lineAndIndex._2 == 0) Text(lineAndIndex._1)
          else <br/> ++ Text(lineAndIndex._1)
        })
        <p>{lines}</p>
      }
    (xml, lineCount)
  }


  /** The Nashorn Javascript engine isn't thread safe. */
  private val threadLocalJavascriptEngine = new jl.ThreadLocal[js.ScriptEngine]


  private def javascriptEngine: js.ScriptEngine = {
    val engineOrNull = threadLocalJavascriptEngine.get
    if (engineOrNull != null)
      return engineOrNull

    val newEngine = new js.ScriptEngineManager(null).getEngineByName("nashorn")
    // Remarkable seems to expect `global` to exist. (Remarkable is a CommonMark Markdown renderer.)
    newEngine.eval("var global = this;")

    def evalFile(path: String) {
      val stream = getClass().getResourceAsStream(path)
      newEngine.eval(new java.io.InputStreamReader(stream))
    }
    evalFile("/public/res/remarkable.min.js")
    evalFile("/public/res/html-sanitizer-bundle.js")
    evalFile("/public/res/mentions-remarkable-plugin.js")

    newEngine.eval(i"""
      |remarkable = new Remarkable({ html: true });
      |remarkable.use(MentionsRemarkablePlugin());
      |
      |function renderCommonMark(source) {
      |  return remarkable.render(source);
      |}
      |""")
    threadLocalJavascriptEngine.set(newEngine)
    newEngine
  }


  /** Converts CommonMark markdown to HTML.
    */
  def markdownToSafeHtml(source: String, hostAndPort: String,
        allowClassIdDataAttrs: Boolean, makeLinksNofollow: Boolean = true): NodeSeq = {
    val htmlTextUnsafe = javascriptEngine.asInstanceOf[js.Invocable].invokeFunction(
      "renderCommonMark", source).asInstanceOf[String]

{'fraction_non_alphanumeric': 0.10284046255753901, 'fraction_numerical': 0.004939934882676546, 'mean_word_length': 3.347486578818936, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5876350', 'n_tokens_mistral': 2752, 'n_tokens_neox': 2593, 'n_words': 819}