<commit_before>require 'yaml'
require 'time'
require 'cgi'


OUT = File.join(__dir__, "../out", "index.html")
IN = File.join(__dir__, "../out", "quotes.yml")
HTML_TPL = File.join(__dir__, "../res", "template.html")
HTML_TPL_Q = File.join(__dir__, "../res", "quote-tpl.html")

def compile_quotes_file(html_tpl, yaml_file)
  template = File.open(html_tpl).readlines().join("\n")
  yml = YAML.load(File.open(yaml_file)).reverse()
  output = ''

  yml.each do |quote|
      escapedQuote = quote["quote"].gsub(/(?<! \|\|) (<[^ >]+>) /, " || \\1 ")
    escapedQuote = CGI.escapeHTML(escapedQuote)
        .gsub("||", "<br>")  # we often use two pipes to denote a newline in quotes.


    output << template.gsub("{{ id }}", quote["id"].to_s)
        .gsub("{{ quote }}", escapedQuote)
        .gsub("{{ adder }}", quote["added_by"].gsub('_', ' '))
        .gsub("{{ time }}", quote["created_at"].iso8601)
        .gsub("{{ deleted }}", (quote["deleted"] ? "true" : "false"))
        .gsub("{{ class-deleted }}", (quote["deleted"] ? "quote-status-deleted" : "quote-status-active"))

  end

  output
end

html = ''
File.open(HTML_TPL).each do |line|
  ln = line.strip()
  if ln == '{{ include template }}'
    html << "\n"
    html << compile_quotes_file(HTML_TPL_Q, IN)
    html << "\n"
    next
  elsif ln == '{{ compile info }}'
    html << '&copy; ' + Time.now.utc.year.to_s + ' &middot; file generated on ' + Time.now.utc.iso8601<commit_msg>further improve logic for timestamp/pipe matching<commit_after>require 'yaml'
require 'time'
require 'cgi'


OUT = File.join(__dir__, "../out", "index.html")
IN = File.join(__dir__, "../out", "quotes.yml")
HTML_TPL = File.join(__dir__, "../res", "template.html")
HTML_TPL_Q = File.join(__dir__, "../res", "quote-tpl.html")

def compile_quotes_file(html_tpl, yaml_file)
  template = File.open(html_tpl).readlines().join("\n")
  yml = YAML.load(File.open(yaml_file)).reverse()
  output = ''

  yml.each do |quote|

    escapedQuote = quote["quote"]
        .gsub(/[\[\(]*[\d+:]+[\]\)]* (<[^ >]+>)/, ' \1')  # remove timestamps
        .gsub(/(?<! \|\|) (<[^ >]+>) /, ' || \1 ') # Prepend || to nicks
    escapedQuote = CGI.escapeHTML(escapedQuote)
        .gsub("||", "<br>")  # we often use two pipes to denote a newline in quotes.


    output << template.gsub("{{ id }}", quote["id"].to_s)
        .gsub("{{ quote }}", escapedQuote)
        .gsub("{{ adder }}", quote["added_by"].gsub('_', ' '))
        .gsub("{{ time }}", quote["created_at"].iso8601)
        .gsub("{{ deleted }}", (quote["deleted"] ? "true" : "false"))
        .gsub("{{ class-deleted }}", (quote["deleted"] ? "quote-status-deleted" : "quote-status-active"))

  end

  output
end

html = ''
File.open(HTML_TPL).each do |line|
  ln = line.strip()
  if ln == '{{ include template }}'
    html << "\n"
    html << compile_quotes_file(HTML_TPL_Q, IN)
    html << "\n"
    next
  elsif ln == '{{ compile info }}'
    html << '&copy; ' + Time.now.utc.year.to_s + ' &middot; file generated on ' + Time.now.utc.iso8601
{'fraction_non_alphanumeric': 0.2312023848956608, 'fraction_numerical': 0.006293474660483604, 'mean_word_length': 3.2715700141442716, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1014817', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1110, 'n_words': 226}