<commit_before>class ManualIndexPage
  ICINGA_ALERTS = "Icinga alerts".freeze

  attr_reader :sitemap

  def initialize(sitemap)
    @sitemap = sitemap
  end

  def columns
    [
      first_column,
      in_two_groups[0],
      in_two_groups[1],
    ]
  end

private

  def first_column
    {
      "Top tasks" => most_important_pages,
      "Manual" => manual_pages.select { |page| page.data.section == "Manual" },
      "Expired pages" => page_freshness.expired_pages,
      "Pages due for review this week" => page_freshness.expiring_soon,
      ICINGA_ALERTS => manual_pages.select { |page| page.data.section == ICINGA_ALERTS },
    }
  end

  def page_freshness
    PageFreshness.new(sitemap)
  end

  def in_two_groups
    slice_size = (manual_pages_grouped_by_section.size / 2.0).ceil
    manual_pages_grouped_by_section.each_slice(slice_size).to_a
  end

  def manual_pages_grouped_by_section
    manual_pages
      .reject { |page| page.data.section.in?([ICINGA_ALERTS, "Manual"]) } # these go into the 1st column
      .group_by { |page| page.data.section || "Uncategorised" }
      .sort_by(&:first)
  end

  def most_important_pages
    manual_pages.select { |page| page.data.important }
  end

  def manual_pages
    sitemap.resources
      .select { |page| page.path.start_with?('manual/') && page.path.end_with?('.html') }
      .sort_by { |page| page.data.title.try(:downcase).to_s }
  end
end<commit_msg>Stop redirects from showing up on manual page<commit_after>class ManualIndexPage
  ICINGA_ALERTS = "Icinga alerts".freeze

  attr_reader :sitemap

  def initialize(sitemap)
    @sitemap = sitemap
  end

  def columns
    [
      first_column,
      in_two_groups[0],
      in_two_groups[1],
    ]
  end

private

  def first_column
    {
      "Top tasks" => most_important_pages,
      "Manual" => manual_pages.select { |page| page.data.section == "Manual" },
      "Expired pages" => page_freshness.expired_pages,
      "Pages due for review this week" => page_freshness.expiring_soon,
      ICINGA_ALERTS => manual_pages.select { |page| page.data.section == ICINGA_ALERTS },
    }
  end

  def page_freshness
    PageFreshness.new(sitemap)
  end

  def in_two_groups
    slice_size = (manual_pages_grouped_by_section.size / 2.0).ceil
    manual_pages_grouped_by_section.each_slice(slice_size).to_a
  end

  def manual_pages_grouped_by_section
    manual_pages
      .reject { |page| page.data.section.in?([ICINGA_ALERTS, "Manual"]) } # these go into the 1st column
      .group_by { |page| page.data.section || "Uncategorised" }
      .sort_by(&:first)
  end

  def most_important_pages
    manual_pages.select { |page| page.data.important }
  end

  def manual_pages
    sitemap.resources
      .select { |page| page.path.start_with?('manual/') && page.path.end_with?('.html') && page.data.title }
      .sort_by { |page| page.data.title.downcase }
  end
end
{'fraction_non_alphanumeric': 0.10957004160887657, 'fraction_numerical': 0.0034674063800277394, 'mean_word_length': 3.274074074074074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3591479', 'n_tokens_mistral': 1065, 'n_tokens_neox': 1026, 'n_words': 193}