<commit_before>module Sitemap

  class Generator

    include Singleton

    SEARCH_ATTRIBUTES = {
      :updated_at       => "lastmod",
      :change_frequency => "changefreq",
      :priority         => "priority"
    }

    attr_accessor :store, :host, :routes, :fragments

    # Instantiates a new object.
    # Should never be called directly.
    def initialize
      self.class.send(:include, Rails.application.routes.url_helpers)
      self.fragments = []
      self.store = Store.new(:max_entries => Sitemap.configuration.max_urls)
      self.store.before_reset do |entries|
        self.process_fragment!
      end
    end

    # Sets the urls to be indexed.
    #
    # The +host+, or any other global option can be set here:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     ...
    #   end
    #
    # Literal paths can be added as follows:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     literal "/some_fancy_url"
    #   end    
    #
    # Simple paths can be added as follows:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     path :faq
    #   end
    #
    # Object collections are supported too:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     resources :activities
    #   end
    #
    # Search options such as frequency and priority can be declared as an options hash:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     path :root, :priority => 1
    #     path :faq, :priority => 0.8, :change_frequency => "daily"
    #     resources :activities, :change_frequency => "weekly"
    #   end
    #
    def load(options = {}, &block)
      options.each do |k, v|
        self.send("#{k}=", v)
      end
      self.routes = block
    end
    
    # Adds the literal url (for consistency, starting with a "/"  as in "/my_url")
    # accepts similar options to path and resources
    def literal(target_url, options = {})
      search = Sitemap.configuration.search.clone.merge!(options.select { |k, v| SEARCH_ATTRIBUTES.keys.include?(k) })
      output_host =  options[:host] || host
      self.store << {
        :url =>"http://#{output_host}#{target_url}",
        :search => search
      }
    end<commit_msg>call procs if passed in to literals<commit_after>module Sitemap

  class Generator

    include Singleton

    SEARCH_ATTRIBUTES = {
      :updated_at       => "lastmod",
      :change_frequency => "changefreq",
      :priority         => "priority"
    }

    attr_accessor :store, :host, :routes, :fragments

    # Instantiates a new object.
    # Should never be called directly.
    def initialize
      self.class.send(:include, Rails.application.routes.url_helpers)
      self.fragments = []
      self.store = Store.new(:max_entries => Sitemap.configuration.max_urls)
      self.store.before_reset do |entries|
        self.process_fragment!
      end
    end

    # Sets the urls to be indexed.
    #
    # The +host+, or any other global option can be set here:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     ...
    #   end
    #
    # Literal paths can be added as follows:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     literal "/some_fancy_url"
    #   end    
    #
    # Simple paths can be added as follows:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     path :faq
    #   end
    #
    # Object collections are supported too:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     resources :activities
    #   end
    #
    # Search options such as frequency and priority can be declared as an options hash:
    #
    #   Sitemap::Generator.instance.load :host => "mywebsite.com" do
    #     path :root, :priority => 1
    #     path :faq, :priority => 0.8, :change_frequency => "daily"
    #     resources :activities, :change_frequency => "weekly"
    #   end
    #
    def load(options = {}, &block)
      options.each do |k, v|
        self.send("#{k}=", v)
      end
      self.routes = block
    end
    
    # Adds the literal url (for consistency, starting with a "/"  as in "/my_url")
    # accepts similar options to path and resources
    def literal(target_url, options = {})
      search = Sitemap.configuration.search.clone.merge!(options.select { |k, v| SEARCH_ATTRIBUTES.keys.include?(k) })
      search.merge!(search) { |type, value| get_data(nil, value) }
            
      output_host =  options[:host] || host
      self.store << {
        :url =>"http://#{output_host}#{target_url}",
        :search => search
      }
    end
{'fraction_non_alphanumeric': 0.1195421788893599, 'fraction_numerical': 0.001271725307333616, 'mean_word_length': 2.277083333333333, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849562', 'n_tokens_mistral': 1492, 'n_tokens_neox': 1475, 'n_words': 380}