<commit_before>module Processor
  def process_text!(element, parent=nil)
    # Skip text nodes that lie outside of elements
    return unless parent
  end
  
  def process_title!(element)
    # Skip title nodes, they are taken care of by their respective parent's processors
  end
  
  def process_section!(element)
    id = element["id"]
    section = @association.owner.sections.create!(:title => element.xpath("title").text, 
                                                  :identifier => id)

    # Process the children elements (paras, etc.) found within a section
    # Need to xpath to section here so that grabs children for that location
    # rather than the section itself
    element.xpath("section").children.each do |child|
      section.elements.send("process_#{child.name}!", child)
    end

    create!(:tag => "section", :identifier => id)
  end
  
  def process_para!(element)
    element = create!(:tag => "para",
                      :identifier => element["id"])
   
  end
  
  def process_formalpara!(element)
    create!(:tag => "formalpara",
           :title => element.xpath("title").text,
           :content => element.to_xml,
           :identifier => element["id"])
  end
  
  def process_itemizedlist!(element)
    list = create!(:tag => "itemizedlist",
            :identifier => element["id"])
    element.xpath("listitem").children.each do |child|
      process_listitem!(list, child)
    end
    list<commit_msg>Commit processor for safekeeping<commit_after>module Processor
  def process_text!(element)
    # Skip text nodes that lie outside of elements
    return if [Section, Chapter].include?(@association.owner.class)
  end
  
  def process_title!(element)
    # Skip title nodes, they are taken care of by their respective parent's processors
  end
  
  def process_section!(element)
    id = element["id"]
    section = @association.owner.sections.create!(:title => element.xpath("title").text, 
                                                  :identifier => id)

    # Process the children elements (paras, etc.) found within a section
    # Need to xpath to section here so that grabs children for that location
    # rather than the section itself
    element.xpath("section").children.each do |child|
      section.elements.send("process_#{child.name}!", child)
    end

    create!(:tag => "section", :identifier => id)
  end
  
  def process_para!(element)
    para = create!(:tag => "para",
                   :identifier => element["id"])
    p element
    element.xpath("para").children.map do |child|
      para.elements.send("process_#{child.name}!", child)
    end
    para
  end
  
  def process_formalpara!(element)
    create!(:tag => "formalpara",
           :title => element.xpath("title").text,
           :content => element.to_xml,
           :identifier => element["id"])
  end
  
  def process_itemizedlist!(element)
    list = create!(:tag => "itemizedlist",
            :identifier => element["id"])
    element.xpath("listitem").children.each do |child|
      process_listitem!(list, child)
    end
    list
{'fraction_non_alphanumeric': 0.12041544952937358, 'fraction_numerical': 0.0, 'mean_word_length': 2.7448359659781287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2606820', 'n_tokens_mistral': 858, 'n_tokens_neox': 843, 'n_words': 239}