<commit_before>#!/usr/bin/env ruby

require "erb"
require "fileutils"
require "redcarpet"
require "syntax"
require "syntax/convertors/html"
require "rscons/version"

def load_file(file_name)
  contents = File.read(file_name)
  contents.gsub(/\$\{include (.*?)\}/) do |match|
    include_file_name = $1
    File.read(include_file_name)
  end
end

class Generator
  class Section
    attr_reader :title
    attr_reader :new_page
    attr_reader :contents
    attr_reader :anchor
    def initialize(title, new_page)
      @title = title
      @new_page = new_page
      @contents = ""
      @anchor = "s" + title.gsub(/[^a-zA-Z0-9]/, "_")
    end
    def append(contents)
      @contents += contents
    end
  end

  def initialize(input, output_file, multi_file)
    @sections = []
    @current_section_number = [0]
    @lines = input.lines
    while @lines.size > 0
      line = @lines.slice!(0)
      if line =~ /^```(.*)$/
        @sections.last.append(gather_code_section($1))
      elsif line =~ /^(#+)(>)?\s*(.*)$/
        level_text, new_page_text, title_text = $1, $2, $3
        level = $1.size
        new_page = !new_page_text.nil?
        section_number = get_next_section_number(level)
        section_title = "#{section_number} #{title_text}"
        @sections << Section.new(section_title, new_page)
        @sections.last.append("#{level_text} #{section_title}")
      elsif @sections.size > 0
        @sections.last.append(line)
      end
    end

    renderer = Redcarpet::Render::HTML.new
    markdown = Redcarpet::Markdown.new(renderer)
    content = %[<h1>Table of Contents</h1>\n]
    @sections.each do |section|
      content += %[<a href="##{section.anchor}">#{section.title}</a><br/>\n]
    end
    @sections.each do |section|
      content += %[<a name="#{section.anchor}" />]
      content += markdown.render(section.contents)
    end

    template = File.read("rb/assets/user_guide.html.erb")
    erb = ERB.new(template, nil, "<>")
    html_result = erb.result(binding.clone)
    File.open(output_file, "w") do |fh|
      fh.write(html_result)
    end
  end

  def gather_code_section(syntax)
    code = ""
    loop do
      line = @lines.slice!(0)
      if line =~ /^```/
        break
      end
      code += line
    end
    if syntax != ""
      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
    else
      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]<commit_msg>user guide: indent table of contents<commit_after>#!/usr/bin/env ruby

require "erb"
require "fileutils"
require "redcarpet"
require "syntax"
require "syntax/convertors/html"
require "rscons/version"

def load_file(file_name)
  contents = File.read(file_name)
  contents.gsub(/\$\{include (.*?)\}/) do |match|
    include_file_name = $1
    File.read(include_file_name)
  end
end

class Generator
  class Section
    attr_reader :number
    attr_reader :title
    attr_reader :new_page
    attr_reader :contents
    attr_reader :anchor
    def initialize(number, title, new_page)
      @number = number
      @title = title.gsub(/[`]/, "")
      @new_page = new_page
      @contents = ""
      @anchor = "s" + title.gsub(/[^a-zA-Z0-9]/, "_")
    end
    def append(contents)
      @contents += contents
    end
  end

  def initialize(input, output_file, multi_file)
    @sections = []
    @current_section_number = [0]
    @lines = input.lines
    while @lines.size > 0
      line = @lines.slice!(0)
      if line =~ /^```(.*)$/
        @sections.last.append(gather_code_section($1))
      elsif line =~ /^(#+)(>)?\s*(.*)$/
        level_text, new_page_text, title_text = $1, $2, $3
        level = $1.size
        new_page = !new_page_text.nil?
        section_number = get_next_section_number(level)
        @sections << Section.new(section_number, title_text, new_page)
        @sections.last.append("#{level_text} #{section_number} #{title_text}")
      elsif @sections.size > 0
        @sections.last.append(line)
      end
    end

    renderer = Redcarpet::Render::HTML.new
    markdown = Redcarpet::Markdown.new(renderer)
    content = %[<h1>Table of Contents</h1>\n]
    @sections.each do |section|
      indent = section.number.split(".").size - 1
      content += %[<span style="padding-left: #{4 * indent}ex;">]
      content += %[<a href="##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
      content += %[</span>]
    end
    @sections.each do |section|
      content += %[<a name="#{section.anchor}" />]
      content += markdown.render(section.contents)
    end

    template = File.read("rb/assets/user_guide.html.erb")
    erb = ERB.new(template, nil, "<>")
    html_result = erb.result(binding.clone)
    File.open(output_file, "w") do |fh|
      fh.write(html_result)
    end
  end

  def gather_code_section(syntax)
    code = ""
    loop do
      line = @lines.slice!(0)
      if line =~ /^```/
        break
      end
      code += line
    end
    if syntax != ""
      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
    else
      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
{'fraction_non_alphanumeric': 0.1639784946236559, 'fraction_numerical': 0.006144393241167435, 'mean_word_length': 2.852810650887574, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2930926', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1809, 'n_words': 347}