<commit_before>//
// markup.go 
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)


// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
var re_external_link = regexp.MustCompile("^(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?$");


func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if strings.HasPrefix(line, "&gt;") {
    // le ebin meme arrows
    markup += "<p><span class='memearrows'>"
    markup += line
    markup += "</span></p>"
  } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
    // redtext
    markup += "<p><span class='redtext'>"
    markup += line[2:len(line)-2]
    markup += "</span></p>"
  } else {
    // regular line
    markup += "<p>"
    // linkify it
    markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
    markup += "</p>"
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "<p>"
  markup += line
  markup += "</p>"
  return
}

func memeposting(src string) (markup string) {
  // escape
  src = html.EscapeString(src)

  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"<commit_msg>try another way to get linkify to work<commit_after>//
// markup.go 
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)


// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
var re_external_link = regexp.MustCompile("^(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?$");


func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if strings.HasPrefix(line, ">") {
    // le ebin meme arrows
    markup += "<p><span class='memearrows'>"
    markup += html.EscapeString(line)
    markup += "</span></p>"
  } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
    // redtext
    markup += "<p><span class='redtext'>"
    markup += html.EscapeString(line[2:len(line)-2])
    markup += "</span></p>"
  } else {
    // regular line
    markup += "<p>"
    // linkify it
    markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
    markup += "</p>"
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "<p>"
  markup += html.EscapeString(line)
  markup += "</p>"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
{'fraction_non_alphanumeric': 0.2895762932154926, 'fraction_numerical': 0.05406810501689628, 'mean_word_length': 3.219298245614035, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 34, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1157727', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1675, 'n_words': 242}