<commit_before># Set the host name for URL creation
SitemapGenerator::Sitemap.default_host = Rails.application.config.sitemap[:host]

SitemapGenerator::Sitemap.create do
  break unless Rails.application.config.sitemap[:generate]

  # Add static pages
  [:en, :da].each do |locale|
    ['', 'search-and-get', 'data', 'faq'].each do |page|
      add "/#{locale}/about/#{page}"
    end
  end

  # Add single record pages
  cursorMark = '*'
  loop do
    response = Blacklight.solr.get('/solr/metastore/ddf_publ', :params => {
      'q'          => '*:*',
      'fl'         => 'cluster_id_ss',
      'cursorMark' => cursorMark,
      'rows'       => ENV['BATCH_SIZE'] || 1000,<commit_msg>exclude unwanted results from sitemap<commit_after># Set the host name for URL creation
SitemapGenerator::Sitemap.default_host = Rails.application.config.sitemap[:host]

SitemapGenerator::Sitemap.create do
  break unless Rails.application.config.sitemap[:generate]

  # Add static pages
  [:en, :da].each do |locale|
    ['', 'search-and-get', 'data', 'faq'].each do |page|
      add "/#{locale}/about/#{page}"
    end
  end

  # Add single record pages
  cursorMark = '*'
  loop do
    response = Blacklight.solr.get('/solr/metastore/ddf_publ', :params => {
      'q'          => '*:*',
      'fl'         => 'cluster_id_ss',
      'fq'         => 'NOT format:person AND NOT source_ss:rdb_ucviden',
      'cursorMark' => cursorMark,
      'rows'       => ENV['BATCH_SIZE'] || 1000,
{'fraction_non_alphanumeric': 0.1601908657123381, 'fraction_numerical': 0.0054533060668029995, 'mean_word_length': 3.1235955056179776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6297207', 'n_tokens_mistral': 517, 'n_tokens_neox': 494, 'n_words': 107}