<commit_before>module Processor
  
  def build_element(markup, name)
    new(:tag        => name,
        :content    => markup.to_html,
        :xml_id     => markup["id"])
  end
  
  def process!(chapter, markup)
    send("process_#{markup.name}!", chapter, markup)
  end

  def process_p!(chapter, markup)
    paragraph = build_element(markup, "p")
    chapter.elements << paragraph
    
    callout_count = 0
    content = markup.to_html
    # Parse footnotes
    markup.css("span.footnote").each do |footnote|
      chapter.footnote_count += 1
      footnote_link = "<a href='#footnote_#{chapter.footnote_count}' class='footnote'><sup>#{chapter.footnote_count}</sup></a>"
      content = content.gsub(/<span class="footnote" id="#{footnote["id"]}">(.*?)<\/span>/, footnote_link)
      process_footnote!(chapter, footnote)
    end
    
    # Parse callouts
    markup.css("span.callout").each do |callout|
      callout_count += 1
      image = "<img src='/images/callouts/#{callout_count}' class='callout'/>"
      content = content.gsub(callout.to_html, image)
    end

    paragraph.content = content
    paragraph
  end
  
  def process_footnote!(chapter, markup)
    # add a footnote tag immediately after the paragraph it is contained within
    chapter.elements << build_element(markup, "footnote")
    # Remove the footnote's elements from the dom, the footnote tag and its containing elements
    [markup, markup.children].flatten.map(&:remove)
  end
  
  def process_section!(chapter, markup)
    title = get_title_element(markup)
    
    # Increment last section count number
    chapter.section_count[-1] += 1
    original_title = title.text
    title.content = "#{chapter.section_count.join(".")} #{title.content}"

    section = new(:tag        => "section",
                  :title      => original_title,
                  :content    => title,
                  :xml_id     => markup["id"],
                  :number     => chapter.section_count[-1])
    # section_count > 2 if this section is a sub-section, 
    # in which case it would be picked up by the `within_section` code beneath
    chapter.elements << section #unless chapter.section_count.size > 2
    within_section(chapter) do
      markup.children.each do |element|
        process!(chapter, element)
      end
    end
    section
  end

  def process_div!(chapter, markup)
    # Divs have classes to identify them, separate them and process them as we see fit
    method = "process_#{markup["class"]}!"
    if self.respond_to?(method)
      send(method, chapter, markup)
    else
      # Or go kaboom if we don't know what the hell it is
      raise "I don't know what to do with div.#{markup["class"]}"
    end
  end
  
  def process_formalpara!(chapter, markup)
    # Create a new element which is just a <h4> of the title
    formalpara = new(:tag        => "formalpara",
                     :xml_id     => markup["id"],
                     :content    => markup.css("h4").to_html)

    chapter.elements << formalpara
    # Iterate through and process each sub-element of the formalpara separately
    markup.children.each { |element| process!(chapter, element) }
    
    formalpara
  end
  
  def process_ul!(chapter, markup)
    chapter.elements << new(:tag        => "ul",
                            :xml_id     => markup["id"],
                            :content    => markup.to_html)
  end
  
  def process_informalexample!(chapter, markup)
    # Remove excessive linebreaks
    pre = markup.css("pre")[0]
    pre.content = pre.content.strip
    informalexample = new(:tag        => "informalexample",
                            :xml_id     => markup["id"],
                            :content    => markup.css("pre").to_html)
    chapter.elements << informalexample
    informalexample
  end
  
  def process_example!(chapter, markup)
    chapter.elements << build_element(markup, "element")
  end
  
  def process_figure!(chapter, markup)
    chapter.figure_count += 1
    filename = markup.css("img")[0]["src"]
    figure = chapter.figures.find_or_initialize_by_filename(filename)
    figure.figure = File.open(chapter.git.path + filename)
    chapter.elements << build_element(markup, "figure")
  end
  
  def process_table!(chapter, markup)
    chapter.elements << build_element(markup, "table")
  end
  
  def process_note!(chapter, markup)
    chapter.elements << build_element(markup, "note")
  end
  
  def process_warning!(chapter, markup)
    chapter.elements << build_element(markup, "warning")
  end
  
  def process_tip!(chapter, markup)
    chapter.elements << build_element(markup, "tip")
  end
  
  def process_indexterm!(chapter, markup)
    # TODO: process at a later date
  end
  
  def process_header!(chapter, markup)
    # headers are processed separately, we don't care about them in this context
  end
  
  # H4s are formal paragraph headers
  def process_h4!(chapter, markup)
    chapter.elements << build_element(markup, "h4")
  end
  
  alias_method :process_h1!, :process_header!
  alias_method :process_h2!, :process_header!
  alias_method :process_h3!, :process_header!
  
  def process_text!(chapter, markup)
    # We don't care about orphaned text, only text within an element such as a paragraph.
  end
  
  private
  
  
  # Used for sub-sections, will make section_count 
  def within_section(chapter)
    chapter.section_count.push(0)
    yield
    chapter.section_count.pop
  end
  
  def get_title_element(markup)
    title = markup.css("h2").first unless markup.css("h2").empty?
    title ||= markup.css("h3").first unless markup.css("h3").empty?
    title
  end
    
end
<commit_msg>Tackle figure storing yet another way<commit_after>module Processor
  
  def build_element(markup, name)
    new(:tag        => name,
        :content    => markup.to_html,
        :xml_id     => markup["id"])
  end
  
  def process!(chapter, markup)
    send("process_#{markup.name}!", chapter, markup)
  end

  def process_p!(chapter, markup)
    paragraph = build_element(markup, "p")
    chapter.elements << paragraph
    
    callout_count = 0
    content = markup.to_html
    # Parse footnotes
    markup.css("span.footnote").each do |footnote|
      chapter.footnote_count += 1
      footnote_link = "<a href='#footnote_#{chapter.footnote_count}' class='footnote'><sup>#{chapter.footnote_count}</sup></a>"
      content = content.gsub(/<span class="footnote" id="#{footnote["id"]}">(.*?)<\/span>/, footnote_link)
      process_footnote!(chapter, footnote)
    end
    
    # Parse callouts
    markup.css("span.callout").each do |callout|
      callout_count += 1
      image = "<img src='/images/callouts/#{callout_count}' class='callout'/>"
      content = content.gsub(callout.to_html, image)
    end

    paragraph.content = content
    paragraph
  end
  
  def process_footnote!(chapter, markup)
    # add a footnote tag immediately after the paragraph it is contained within
    chapter.elements << build_element(markup, "footnote")
    # Remove the footnote's elements from the dom, the footnote tag and its containing elements
    [markup, markup.children].flatten.map(&:remove)
  end
  
  def process_section!(chapter, markup)
    title = get_title_element(markup)
    
    # Increment last section count number
    chapter.section_count[-1] += 1
    original_title = title.text
    title.content = "#{chapter.section_count.join(".")} #{title.content}"

    section = new(:tag        => "section",
                  :title      => original_title,
                  :content    => title,
                  :xml_id     => markup["id"],
                  :number     => chapter.section_count[-1])
    # section_count > 2 if this section is a sub-section, 
    # in which case it would be picked up by the `within_section` code beneath
    chapter.elements << section #unless chapter.section_count.size > 2
    within_section(chapter) do
      markup.children.each do |element|
        process!(chapter, element)
      end
    end
    section
  end

  def process_div!(chapter, markup)
    # Divs have classes to identify them, separate them and process them as we see fit
    method = "process_#{markup["class"]}!"
    if self.respond_to?(method)
      send(method, chapter, markup)
    else
      # Or go kaboom if we don't know what the hell it is
      raise "I don't know what to do with div.#{markup["class"]}"
    end
  end
  
  def process_formalpara!(chapter, markup)
    # Create a new element which is just a <h4> of the title
    formalpara = new(:tag        => "formalpara",
                     :xml_id     => markup["id"],
                     :content    => markup.css("h4").to_html)

    chapter.elements << formalpara
    # Iterate through and process each sub-element of the formalpara separately
    markup.children.each { |element| process!(chapter, element) }
    
    formalpara
  end
  
  def process_ul!(chapter, markup)
    chapter.elements << new(:tag        => "ul",
                            :xml_id     => markup["id"],
                            :content    => markup.to_html)
  end
  
  def process_informalexample!(chapter, markup)
    # Remove excessive linebreaks
    pre = markup.css("pre")[0]
    pre.content = pre.content.strip
    informalexample = new(:tag        => "informalexample",
                            :xml_id     => markup["id"],
                            :content    => markup.css("pre").to_html)
    chapter.elements << informalexample
    informalexample
  end
  
  def process_example!(chapter, markup)
    chapter.elements << build_element(markup, "element")
  end
  
  def process_figure!(chapter, markup)
    chapter.figure_count += 1
    filename = markup.css("img")[0]["src"]
    figure = chapter.figures.where(:filename => filename).first
    figure ||= chapter.figures.build(:filename => filename)
    figure.figure = File.open(chapter.git.path + filename)
    chapter.elements << build_element(markup, "figure")
  end
  
  def process_table!(chapter, markup)
    chapter.elements << build_element(markup, "table")
  end
  
  def process_note!(chapter, markup)
    chapter.elements << build_element(markup, "note")
  end
  
  def process_warning!(chapter, markup)
    chapter.elements << build_element(markup, "warning")
  end
  
  def process_tip!(chapter, markup)
    chapter.elements << build_element(markup, "tip")
  end
  
  def process_indexterm!(chapter, markup)
    # TODO: process at a later date
  end
  
  def process_header!(chapter, markup)
    # headers are processed separately, we don't care about them in this context
  end
  
  # H4s are formal paragraph headers
  def process_h4!(chapter, markup)
    chapter.elements << build_element(markup, "h4")
  end
  
  alias_method :process_h1!, :process_header!
  alias_method :process_h2!, :process_header!
  alias_method :process_h3!, :process_header!
  
  def process_text!(chapter, markup)
    # We don't care about orphaned text, only text within an element such as a paragraph.
  end
  
  private
  
  
  # Used for sub-sections, will make section_count 
  def within_section(chapter)
    chapter.section_count.push(0)
    yield
    chapter.section_count.pop
  end
  
  def get_title_element(markup)
    title = markup.css("h2").first unless markup.css("h2").empty?
    title ||= markup.css("h3").first unless markup.css("h3").empty?
    title
  end
    
end

{'fraction_non_alphanumeric': 0.1134438955539873, 'fraction_numerical': 0.004234297812279464, 'mean_word_length': 2.824898785425101, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5180723', 'n_tokens_mistral': 3550, 'n_tokens_neox': 3371, 'n_words': 875}