<commit_before>#!/usr/bin/env ruby
require 'nokogiri'
require 'rake'
require 'fileutils'

module Rake
  class BlacklightSitemapTask
    # base url used for all locations
    attr_accessor :url

    # base filename to use for sitemap in case these will be moved to a location
    # that hosts other sitemaps so these sitemaps do not overwrite others
    attr_accessor :base_filename

    # should the files be gzipped? requires the commandline tool gzip
    attr_accessor :gzip

    # value for changefreq for each page listed
    attr_accessor :changefreq

    # the most resources which should be listed within a single sitemap
    # defaults to 50,000
    attr_accessor :max

    # Solr field that contains a date to create a lastmod date for the page.
    # Currently must be a string as in W3C Datetime format or YYYY-MM-DD
    attr_accessor :lastmod_field

    # Solr field to use to provide a priority for this resource
    attr_accessor :priority_field

    # Solr sort option
    attr_accessor :sort

    def initialize
      @url = 'http://localhost:3000/catalog'
      @base_filename = 'blacklight'
      @gzip = false
      @changefreq = nil
      @max = 50000 #default value for max number of locs per sitemap file
      @lastmod_field = 'timestamp'
      @priority_field = nil
      @sort = '_docid_ asc'
      yield self if block_given?<commit_msg>comment on the default sort option<commit_after>#!/usr/bin/env ruby
require 'nokogiri'
require 'rake'
require 'fileutils'

module Rake
  class BlacklightSitemapTask
    # base url used for all locations
    attr_accessor :url

    # base filename to use for sitemap in case these will be moved to a location
    # that hosts other sitemaps so these sitemaps do not overwrite others
    attr_accessor :base_filename

    # should the files be gzipped? requires the commandline tool gzip
    attr_accessor :gzip

    # value for changefreq for each page listed
    attr_accessor :changefreq

    # the most resources which should be listed within a single sitemap
    # defaults to 50,000
    attr_accessor :max

    # Solr field that contains a date to create a lastmod date for the page.
    # Currently must be a string as in W3C Datetime format or YYYY-MM-DD
    attr_accessor :lastmod_field

    # Solr field to use to provide a priority for this resource
    attr_accessor :priority_field

    # Solr sort option
    attr_accessor :sort

    def initialize
      @url = 'http://localhost:3000/catalog'
      @base_filename = 'blacklight'
      @gzip = false
      @changefreq = nil
      @max = 50000 #default value for max number of locs per sitemap file
      @lastmod_field = 'timestamp'
      @priority_field = nil
      @sort = '_docid_ asc' # http://osdir.com/ml/solr-user.lucene.apache.org/2010-03/msg01371.html
      yield self if block_given?
{'fraction_non_alphanumeric': 0.054313099041533544, 'fraction_numerical': 0.01455449059282925, 'mean_word_length': 3.0663780663780664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455295', 'n_tokens_mistral': 873, 'n_tokens_neox': 818, 'n_words': 333}