<commit_before>module Sitemap::Addon
  module Body
    extend ActiveSupport::Concern
    extend SS::Addon

    set_order 200

    included do
      field :sitemap_urls, type: SS::Extensions::Lines
      field :sitemap_depth, type: Integer, default: 3
      field :sitemap_page_state, type: String, default: "hide"
      field :sitemap_deny_urls, type: SS::Extensions::Lines
      permit_params :sitemap_urls, :sitemap_depth, :sitemap_page_state, :sitemap_deny_urls

      after_generate_file :generate_sitemap_xml
      after_remove_file :remove_sitemap_xml
      after_rename_file :rename_sitemap_xml
    end

    public
      def sitemap_xml_path
        path.sub(/\.[^\/]+$/, ".xml")
      end

      def sitemap_depth
        value = self[:sitemap_depth].to_i
        (value < 1 || 5 < value) ? 5 : value
      end

      def sitemap_page_state_options
        [
          [I18n.t('views.options.state.show'), 'show'],
          [I18n.t('views.options.state.hide'), 'hide'],
        ]
      end

      def load_sitemap_urls(opts = {})
        entries = Cms::Node.where(site_id: site_id).public.
          where(:depth.lte => sitemap_depth).
          entries

        if sitemap_page_state != "hide"
          entries += Cms::Page.where(site_id: site_id).public.
            where(:depth.lte => sitemap_depth).
            not(filename: /\/index\.html$/).
            entries
        end

        # deny
        if sitemap_deny_urls.present?
          regex = sitemap_deny_urls.map { |m| /^\/?#{m}/ }
          regex = Regexp.union(regex)
          entries  = entries.reject { |e| e.url =~ regex }
        end

        # sort by order
        h = {}
        entries.each do |e|
          parent_url = e.parent ? e.parent.url : "/"
          h[parent_url] ||= []
          h[parent_url] << e
        end
        entries = h.map { |k, v| v.sort_by(&:order) }.flatten

        urls = entries.map { |m| opts[:name] ? "#{m.url} ##{m.name}" : m.url }
        urls
      end

      def sitemap_list
        list = []
        urls = sitemap_urls.presence || load_sitemap_urls(name: true)
        urls.each do |url|
          next if url.strip.blank?

          depth = url.scan(/[^\/]+/).size
          next if depth > sitemap_depth

          if url =~ /#/
            data = { url: url.sub(/\s*#.*/, ""), name: url.sub(/^.*?\s*#/, ""), depth: depth }
          else
            url   = url.strip.sub(/\/$/, "")
            model = url =~ /(^|\/)[^\.]+$/ ? Cms::Node : Cms::Page

            if item = model.where(site_id: site_id).public.filename(url).first
              data = { url: item.url, name: item.name, depth: depth }
            else
              data = { url: url, name: url, depth: depth }
            end
          end

          if data[:depth] < sitemap_depth
            list << data
          else
            next if list.blank?<commit_msg>sitemap: fix sort in load_sitemap_urls #374<commit_after>module Sitemap::Addon
  module Body
    extend ActiveSupport::Concern
    extend SS::Addon

    set_order 200

    included do
      field :sitemap_urls, type: SS::Extensions::Lines
      field :sitemap_depth, type: Integer, default: 3
      field :sitemap_page_state, type: String, default: "hide"
      field :sitemap_deny_urls, type: SS::Extensions::Lines
      permit_params :sitemap_urls, :sitemap_depth, :sitemap_page_state, :sitemap_deny_urls

      after_generate_file :generate_sitemap_xml
      after_remove_file :remove_sitemap_xml
      after_rename_file :rename_sitemap_xml
    end

    public
      def sitemap_xml_path
        path.sub(/\.[^\/]+$/, ".xml")
      end

      def sitemap_depth
        value = self[:sitemap_depth].to_i
        (value < 1 || 5 < value) ? 5 : value
      end

      def sitemap_page_state_options
        [
          [I18n.t('views.options.state.show'), 'show'],
          [I18n.t('views.options.state.hide'), 'hide'],
        ]
      end

      def load_sitemap_urls(opts = {})
        entries = Cms::Node.where(site_id: site_id).public.
          where(:depth.lte => sitemap_depth).
          order_by(filename: 1).
          entries

        if sitemap_page_state != "hide"
          entries += Cms::Page.where(site_id: site_id).public.
            where(:depth.lte => sitemap_depth).
            not(filename: /\/index\.html$/).
            order_by(filename: 1).
            entries
        end

        # deny
        if sitemap_deny_urls.present?
          regex = sitemap_deny_urls.map { |m| /^\/?#{m}/ }
          regex = Regexp.union(regex)
          entries  = entries.reject { |e| e.url =~ regex }
        end

        # sort by order
        tree = {}
        def tree.flatten(url, entries)
          return unless self[url]
          self[url].each do |e|
            entries << e
            self.flatten(e.url, entries)
          end
        end

        entries.each do |e|
          parent_url = e.parent ? e.parent.url : "/"
          tree[parent_url] ||= []
          tree[parent_url] << e
        end
        tree.each_value { |v| v.sort_by!(&:order) }
        entries = []
        tree.flatten("/", entries)

        urls = entries.map { |m| opts[:name] ? "#{m.url} ##{m.name}" : m.url }
        urls
      end

      def sitemap_list
        list = []
        urls = sitemap_urls.presence || load_sitemap_urls(name: true)
        urls.each do |url|
          next if url.strip.blank?

          depth = url.scan(/[^\/]+/).size
          next if depth > sitemap_depth

          if url =~ /#/
            data = { url: url.sub(/\s*#.*/, ""), name: url.sub(/^.*?\s*#/, ""), depth: depth }
          else
            url   = url.strip.sub(/\/$/, "")
            model = url =~ /(^|\/)[^\.]+$/ ? Cms::Node : Cms::Page

            if item = model.where(site_id: site_id).public.filename(url).first
              data = { url: item.url, name: item.name, depth: depth }
            else
              data = { url: url, name: url, depth: depth }
            end
          end

          if data[:depth] < sitemap_depth
            list << data
          else
            next if list.blank?
{'fraction_non_alphanumeric': 0.13680567001813088, 'fraction_numerical': 0.0044503049283006425, 'mean_word_length': 1.7606915377616015, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3366515', 'n_tokens_mistral': 2053, 'n_tokens_neox': 2018, 'n_words': 417}