<commit_before>package asciiblog

import java.lang.StringBuilder
import Character._

case class MarkupBlock(
  val startToken: String, val endToken: String,
  val startTag: String,   val endTag: String,
  val recur: Boolean,
  val tight: Boolean,
  val doubleQuotes: Boolean = false,
  val compact: Boolean = false
) {
  val startChar = startToken.charAt(0)
  val startLong = startToken.length > 1

  val endChar   = endToken.charAt(0)
  val endLong   = endToken.length > 1

  def startLen = startToken.length
  def endLen   = endToken.length

  val attributePos = startTag.indexOf('>')

  lazy val asCompact = if (tight) copy(compact = true) else sys.error("non tight blocks cannot have a compact version")

  // long mode for multiple words
  // non-alpha-num START non-whitspace ... non-whitespace END non-alpha-num
  //
  // compact mode for single word
  // START non-whitespace* END


  def checkStart(str: String, i: Int) =
    str.regionMatches(i, startToken, 0, startToken.length)

  def isLongStart(str: String, i: Int) =
    if (!tight) true else (i == 0 || !isLetterOrDigit(str(i-1))) && (i+startLen >= str.length || !isWhitespace(str(i+startLen)))

  def isCompactStart(str: String, i: Int) =
    i+startLen == str.length || !isWhitespace(str(i+startLen))

  def isStart(str: String, i: Int) =
    isLongStart(str, i) || isCompactStart(str, i)


  def checkEnd(str: String, i: Int) =
    str.regionMatches(i, endToken, 0, endToken.length)

  def isLongEnd(str: String, i: Int) =
    if (!tight) true else (i == 0 || !isWhitespace(str(i-1))) && (i+endLen >= str.length || !isLetterOrDigit(str(i+endLen)))

  def isCompactEnd(str: String, i: Int) =
    i == 0 || !isWhitespace(str(i-1))

  def isEnd(str: String, i: Int) =
    isLongEnd(str, i) || isCompactEnd(str, i)


  def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
    require(!recur)
    sb.append(str, from, to)
  }

  override def toString = s"MarkupBlock($startToken...$endToken, ${if (compact) "compact" else "long"})"
}





object MarkupParser {
  val blocks = Array(
    new MarkupBlock("<", ">", "<",   ">",             recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = sb.append(str, from, to)
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && validTags.contains(detectTagName(str, i+1))

      def detectTagName(str: String, from: Int) = {
        var f = from
        var i = from
        if (i < str.length && str.charAt(i) == '/') { f += 1; i += 1 }
        while (i < str.length && isLetterOrDigit(str.charAt(i))) i += 1
        str.substring(f, i)
      }
    },
    new MarkupBlock("<!--", "-->", "",    "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = sb
    },
    new MarkupBlock("[|", "|]", "",       "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
        if (!plaintext) {
          val len = to - from - 2
          for (i <- 0 until (len/5)) { sb.append("█████").append("<wbr>") }
        }
        sb.append("█████")
      }
    },
    new MarkupBlock("`",  "`",  "<code>", "</code>", recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) =
        html.escape(str, from, to, sb)
    },
    new MarkupBlock("''", "''", "",       "",        recur = false, tight = false),
    new MarkupBlock("**", "**", "<b>",    "</b>",    recur = true, tight = true),
    new MarkupBlock("*",  "*",  "<i>",    "</i>",    recur = true, tight = true),
    new MarkupBlock("//", "//", "<i>",    "</i>",    recur = true, tight = true) {
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && (i == 0 || str.charAt(i-1) != ':') // check for https://
      override def checkEnd  (str: String, i: Int) = super.checkEnd  (str, i) && (i == 0 || str.charAt(i-1) != ':')
    },
    new MarkupBlock("\"", "\"", "",       "",        recur = true, tight = true, doubleQuotes = true),
    new MarkupBlock("[[", "]]", "",       "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
        if (!plaintext) {
          sb.append("<a href=").append("#fn"+str.substring(from, to)).append("><sup>").append(str, from, to).append("</sup></a> ")
        }
        sb
      }
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && (i+4 < str.length && Character.isDigit(str.charAt(i+2)))
    },
  )<commit_msg>markup: inline citations<commit_after>package asciiblog

import java.lang.StringBuilder
import Character._

case class MarkupBlock(
  val startToken: String, val endToken: String,
  val startTag: String,   val endTag: String,
  val recur: Boolean,
  val tight: Boolean,
  val doubleQuotes: Boolean = false,
  val compact: Boolean = false
) {
  val startChar = startToken.charAt(0)
  val startLong = startToken.length > 1

  val endChar   = endToken.charAt(0)
  val endLong   = endToken.length > 1

  def startLen = startToken.length
  def endLen   = endToken.length

  val attributePos = startTag.indexOf('>')

  lazy val asCompact = if (tight) copy(compact = true) else sys.error("non tight blocks cannot have a compact version")

  // long mode for multiple words
  // non-alpha-num START non-whitspace ... non-whitespace END non-alpha-num
  //
  // compact mode for single word
  // START non-whitespace* END


  def checkStart(str: String, i: Int) =
    str.regionMatches(i, startToken, 0, startToken.length)

  def isLongStart(str: String, i: Int) =
    if (!tight) true else (i == 0 || !isLetterOrDigit(str(i-1))) && (i+startLen >= str.length || !isWhitespace(str(i+startLen)))

  def isCompactStart(str: String, i: Int) =
    i+startLen == str.length || !isWhitespace(str(i+startLen))

  def isStart(str: String, i: Int) =
    isLongStart(str, i) || isCompactStart(str, i)


  def checkEnd(str: String, i: Int) =
    str.regionMatches(i, endToken, 0, endToken.length)

  def isLongEnd(str: String, i: Int) =
    if (!tight) true else (i == 0 || !isWhitespace(str(i-1))) && (i+endLen >= str.length || !isLetterOrDigit(str(i+endLen)))

  def isCompactEnd(str: String, i: Int) =
    i == 0 || !isWhitespace(str(i-1))

  def isEnd(str: String, i: Int) =
    isLongEnd(str, i) || isCompactEnd(str, i)


  def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
    require(!recur)
    sb.append(str, from, to)
  }

  override def toString = s"MarkupBlock($startToken...$endToken, ${if (compact) "compact" else "long"})"
}





object MarkupParser {
  val blocks = Array(
    new MarkupBlock("<", ">", "<",   ">",             recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = sb.append(str, from, to)
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && validTags.contains(detectTagName(str, i+1))

      def detectTagName(str: String, from: Int) = {
        var f = from
        var i = from
        if (i < str.length && str.charAt(i) == '/') { f += 1; i += 1 }
        while (i < str.length && isLetterOrDigit(str.charAt(i))) i += 1
        str.substring(f, i)
      }
    },
    new MarkupBlock("<!--", "-->", "",    "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = sb
    },
    new MarkupBlock("[|", "|]", "",       "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
        if (!plaintext) {
          val len = to - from - 2
          for (i <- 0 until (len/5)) { sb.append("█████").append("<wbr>") }
        }
        sb.append("█████")
      }
    },
    new MarkupBlock("`",  "`",  "<code>", "</code>", recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) =
        html.escape(str, from, to, sb)
    },
    new MarkupBlock("''", "''", "",       "",        recur = false, tight = false),
    new MarkupBlock(">>", "<<", "<q>",    "</q>",    recur = true, tight = true),
    new MarkupBlock("**", "**", "<b>",    "</b>",    recur = true, tight = true),
    new MarkupBlock("*",  "*",  "<i>",    "</i>",    recur = true, tight = true),
    new MarkupBlock("//", "//", "<i>",    "</i>",    recur = true, tight = true) {
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && (i == 0 || str.charAt(i-1) != ':') // check for https://
      override def checkEnd  (str: String, i: Int) = super.checkEnd  (str, i) && (i == 0 || str.charAt(i-1) != ':')
    },
    new MarkupBlock("\"", "\"", "",       "",        recur = true, tight = true, doubleQuotes = true),
    new MarkupBlock("[[", "]]", "",       "",        recur = false, tight = false) {
      override def appendBody(str: String, from: Int, to: Int, sb: StringBuilder, plaintext: Boolean) = {
        if (!plaintext) {
          sb.append("<a href=").append("#fn"+str.substring(from, to)).append("><sup>").append(str, from, to).append("</sup></a> ")
        }
        sb
      }
      override def checkStart(str: String, i: Int) = super.checkStart(str, i) && (i+4 < str.length && Character.isDigit(str.charAt(i+2)))
    },
  )
{'fraction_non_alphanumeric': 0.18280354351224595, 'fraction_numerical': 0.005211047420531527, 'mean_word_length': 2.9408624229979465, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1027111', 'n_tokens_mistral': 3320, 'n_tokens_neox': 3248, 'n_words': 890}