<commit_before>class Whatsa::Article
  attr_accessor :sections
  attr_reader :contents, :title

  def initialize(noko_doc)
    @title = noko_doc.css('h1').text
    @contents = noko_doc.css('#mw-content-text').children
    @sections = make_sections
  end

  def summary
    self.sections.first.summary
  end

  def full_text
    # name might be a little confusing: it's not really the "full text" of the
    # article, it's the full text of the article summary. I'm naming it #full_text
    # for duck-typing reasons
    self.sections.first.full_text
  end

  def section_titles
    self.sections.map { |s| s.title }
  end

  def choose_section(choice)
    if choice.to_i > 0
      self.sections[choice.to_i - 1]
    else
      get_section_by_title(title)
    end
  end

  private

  def get_section_by_title(title)
    self.sections.find { |s| s.title.downcase == title.downcase }
  end

  def intro_pars
    breakpoint = self.contents.to_a.index { |element| element.name == 'h2' }
    breakpoint ||= -1 # breakpoint would be nil for zero-section articles
    pars = self.contents[0...breakpoint].css('p').map { |par| par.text }
    pars.reject { |par| par == "" }
  end

  def make_sections
    indices = section_indices
    indices << -1
    secs = [Whatsa::Section.new("#{self.title} - Introduction", intro_pars)]
    indices.each_cons(2) do |i, j|
      title = self.contents[i].text.gsub('[edit]', '').strip
      par_nodes = self.contents[i...j].select { |e| e.name == 'p' && e.text != "" }
      pars = par_nodes.map { |par| par.text }
      secs << Whatsa::Section.new(title, pars).tap { |s| s.article = self }
    end
    secs
  end

  def section_indices
    indices = []
    self.contents.each_with_index do |element, i|
      indices << i if element.name == 'h2'
    end
    indices
  end
end<commit_msg>allow <ul> to count as a paragraph when making sections<commit_after>class Whatsa::Article
  attr_accessor :sections
  attr_reader :contents, :title

  def initialize(noko_doc)
    @title = noko_doc.css('h1').text
    @contents = noko_doc.css('#mw-content-text').children
    @sections = make_sections
  end

  def summary
    self.sections.first.summary
  end

  def full_text
    # name might be a little confusing: it's not really the "full text" of the
    # article, it's the full text of the article summary. I'm naming it #full_text
    # for duck-typing reasons
    self.sections.first.full_text
  end

  def section_titles
    self.sections.map { |s| s.title }
  end

  def choose_section(choice)
    if choice.to_i > 0
      self.sections[choice.to_i - 1]
    else
      get_section_by_title(title)
    end
  end

  private

  def get_section_by_title(title)
    self.sections.find { |s| s.title.downcase == title.downcase }
  end

  def intro_pars
    breakpoint = self.contents.to_a.index { |element| element.name == 'h2' }
    breakpoint ||= -1 # breakpoint would be nil for zero-section articles
    pars = self.contents[0...breakpoint].css('p').map { |par| par.text }
    pars.reject { |par| par == "" }
  end

  def make_sections
    indices = section_indices
    indices << -1
    secs = [Whatsa::Section.new("#{self.title} - Introduction", intro_pars)]
    indices.each_cons(2) do |i, j|
      title = self.contents[i].text.gsub('[edit]', '').strip
      par_nodes = self.contents[i...j].select do |e|
        e.name == 'p' || e.name == 'ul' && e.text != ""
      end
      pars = par_nodes.map { |par| par.text.strip }
      secs << Whatsa::Section.new(title, pars).tap { |s| s.article = self }
    end
    secs
  end

  def section_indices
    indices = []
    self.contents.each_with_index do |element, i|
      indices << i if element.name == 'h2'
    end
    indices
  end
end
{'fraction_non_alphanumeric': 0.126514131897712, 'fraction_numerical': 0.004845222072678331, 'mean_word_length': 2.8388429752066116, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '314911', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1251, 'n_words': 322}