<commit_before>if ARGV.length == 0 then
  puts "requires an input file"
  exit 1
end

file = File.open(ARGV[0]).read.split("\n")

output = ARGV[1] if ARGV.length > 1

result = []

file.each do |line|
  append = ""
  type = line.split(" ").first
  if line.strip.length == 0
    result << ""
    next
  end

  content = line.split(" ")[1..-1].join(" ")

  # Image as the main attraction
  if type == "@^" then
    content = "<img src=\"pictures/#{content}\" style=\"margin-top: -50px\" />"
    type = ""
  end
  

  # Shortcut for images. Assumes assets live in pictures/
  if type == "@" then
    content = "<img src=\"pictures/#{content}\" />"
    type.gsub!("@","")
  end

  # Ignore generic line separators
  if type == "---" then
    result << type
    next
  end

  # Fragment
  if type.include? "-" then
    append = " <!-- .element: class=\"fragment\" -->"
    type.gsub!("-","")
  end

  # h1 defaulting to top with padding
  if type == "#" then
    append = " <!-- .element: style=\"margin-bottom: 1em\" -->"
  end
  
  # Center
  if type.include? "=" then
    append = " <!-- .slide: class=\"center\" -->"
    type.gsub!("=","")
  end

  # h0
  if ["!#","!#="].include? type then
    append += " <!-- .element style=\"font-size: 4em\" --> "
    type.gsub!("!","")
  end

  r = "#{type} #{content}#{append}"
  result << r
end

if output
  File.open(output, "w") { |f| f.write result.join("\n") }
  puts "Outputted parsed markdown to #{output}"
else 
  puts result.join("\n")
end

<commit_msg>Fragments come in many shapes<commit_after>if ARGV.length == 0 then
  puts "requires an input file"
  exit 1
end

file = File.open(ARGV[0]).read.split("\n")

output = ARGV[1] if ARGV.length > 1

result = []

file.each do |line|
  append = ""
  type = line.split(" ").first
  if line.strip.length == 0
    result << ""
    next
  end

  content = line.split(" ")[1..-1].join(" ")

  # Image as the main attraction
  if type == "@^" then
    content = "<img src=\"pictures/#{content}\" style=\"margin-top: -50px\" />"
    type = ""
  end
  

  # Shortcut for images. Assumes assets live in pictures/
  if type == "@" then
    content = "<img src=\"pictures/#{content}\" />"
    type.gsub!("@","")
  end

  # Ignore generic line separators
  if type == "---" then
    result << type
    next
  end

  # Fragment
  if type.end_with? "-" then
    append = " <!-- .element: class=\"fragment\" -->"
    type.gsub!("-","")
  end

  # h1 defaulting to top with padding
  if type == "#" then
    append = " <!-- .element: style=\"margin-bottom: 1em\" -->"
  end
  
  # Center
  if type.include? "=" then
    append = " <!-- .slide: class=\"center\" -->"
    type.gsub!("=","")
  end

  # h0
  if ["!#","!#="].include? type then
    append += " <!-- .element style=\"font-size: 4em\" --> "
    type.gsub!("!","")
  end

  r = "#{type} #{content}#{append}"
  result << r
end

if output
  File.open(output, "w") { |f| f.write result.join("\n") }
  puts "Outputted parsed markdown to #{output}"
else 
  puts result.join("\n")
end


{'fraction_non_alphanumeric': 0.2114683460391117, 'fraction_numerical': 0.009280742459396751, 'mean_word_length': 2.505226480836237, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3890626', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1081, 'n_words': 292}