<commit_before><% cache true do -%>
<% if ENV['PUBLIC_HOSTNAME'] == 'bestpractices.coreinfrastructure.org' -%>
# This is the real production system for the CII Best Practices badge.
<%
  locale_prefixes = I18n.available_locales.map { |loc| "/#{loc}" }
  locale_prefixes_except_en = locale_prefixes.reject { |loc| loc == '/en' }
  all_prefixes = locale_prefixes.prepend('') # ['', '/en', 'zh-CN', ...]
-%>

User-Agent: *
#
# Tell crawlers to skip public user data, to help maintain user privacy.
# With these instructions, if users delete their accounts,
# the user data will rapidly disappear elsewhere as well.
# We don't use wildcards here, because some crawlers don't support wildcards;
# instead we list every supported locale.
<% all_prefixes.each do |prefix| %>
Disallow: <%= prefix %>/users
<% end %>
#
# We want people to have access to the JSON files, so we (now) allow
# crawling the norm /en/projects.json and /en/projects/*.json files.
# However, it's pointless to get them in different locales, so
# only crawl one locale, namely English (en), and the "no locale" version.
<% locale_prefixes_except_en.each do |prefix| %>
Disallow: <%= prefix %>/projects*.json$
<% end %>
#
# We *want* people to crawl [/LOC]/project_stats[.json]
# but there's little value in crawling the per-day statistics in
# [/LOC]/project_stats/[*.json].
# All of that data is in the full downloadable dataset.
# Let's save the site a lot of time by skipping those.
<% all_prefixes.each do |prefix| %>
Disallow: <%= prefix %>/project_stats/
<% end %>
#
# Match with config/initialization
#
Disallow: /admin
Disallow: /backup
Disallow: /cgi
Disallow: /command
Disallow: /common
Disallow: /config
Disallow: /data
Disallow: /dbadmin
Disallow: /dump
Disallow: /error_message
Disallow: /install
Disallow: /joomla
Disallow: /muieblackcat
Disallow: /myadmin
Disallow: /mysql
Disallow: /onvif
Disallow: /options
Disallow: /phpadmin
Disallow: /phpmanager
Disallow: /phpmyadmin
Disallow: /phpMyAdmin
Disallow: /PHPMYADMIN
Disallow: /scripts
Disallow: /setup
Disallow: /sqladmin
Disallow: /sql-admin
Disallow: /submitticket
Disallow: /temp
Disallow: /upload
Disallow: /w00tw00t
Disallow: /webadmin
Disallow: /wootwoot
Disallow: /WootWoot
Disallow: /WooTWooT
Disallow: /wp-
Disallow: /xmlrpc
#
#
# Don't crawl sorts or most queries.
# They have the same data, and crawling all the variants will produce
# slow crawl times.<commit_msg>Modify robots.txt to disallow /projects*[&%] (#1581)<commit_after><% cache true do -%>
<% if ENV['PUBLIC_HOSTNAME'] == 'bestpractices.coreinfrastructure.org' -%>
# This is the real production system for the CII Best Practices badge.
<%
  locale_prefixes = I18n.available_locales.map { |loc| "/#{loc}" }
  locale_prefixes_except_en = locale_prefixes.reject { |loc| loc == '/en' }
  all_prefixes = locale_prefixes.prepend('') # ['', '/en', 'zh-CN', ...]
-%>

User-Agent: *
#
# Tell crawlers to skip public user data, to help maintain user privacy.
# With these instructions, if users delete their accounts,
# the user data will rapidly disappear elsewhere as well.
# We don't use wildcards here, because some crawlers don't support wildcards;
# instead we list every supported locale.
<% all_prefixes.each do |prefix| %>
Disallow: <%= prefix %>/users
<% end %>
#
# We want people to have access to the JSON files, so we (now) allow
# crawling the norm /en/projects.json and /en/projects/*.json files.
# However, it's pointless to get them in different locales, so
# only crawl one locale, namely English (en), and the "no locale" version.
<% locale_prefixes_except_en.each do |prefix| %>
Disallow: <%= prefix %>/projects*.json$
<% end %>
#
# We *want* people to crawl [/LOC]/project_stats[.json]
# but there's little value in crawling the per-day statistics in
# [/LOC]/project_stats/[*.json].
# All of that data is in the full downloadable dataset.
# Let's save the site a lot of time by skipping those.
<% all_prefixes.each do |prefix| %>
Disallow: <%= prefix %>/project_stats/
<% end %>
#
# Match with config/initialization
#
Disallow: /admin
Disallow: /backup
Disallow: /cgi
Disallow: /command
Disallow: /common
Disallow: /config
Disallow: /data
Disallow: /dbadmin
Disallow: /dump
Disallow: /error_message
Disallow: /install
Disallow: /joomla
Disallow: /muieblackcat
Disallow: /myadmin
Disallow: /mysql
Disallow: /onvif
Disallow: /options
Disallow: /phpadmin
Disallow: /phpmanager
Disallow: /phpmyadmin
Disallow: /phpMyAdmin
Disallow: /PHPMYADMIN
Disallow: /scripts
Disallow: /setup
Disallow: /sqladmin
Disallow: /sql-admin
Disallow: /submitticket
Disallow: /temp
Disallow: /upload
Disallow: /w00tw00t
Disallow: /webadmin
Disallow: /wootwoot
Disallow: /WootWoot
Disallow: /WooTWooT
Disallow: /wp-
Disallow: /xmlrpc

# Don't crawl /projects URLs with "&" or "%".
# We get many attacks that use these characters, and in some cases
# temporarily block sites that use them in ways that look like attacks.
# There's no need to crawl them, and by saying "do not crawl them"
# we protect web crawlers from accidentally looking like attackers.
# Ideally we'd only prevent crawling if these were not part of the query,
# but robots.txt doesn't have any way to do that.
<% all_prefixes.each do |prefix| %>
Disallow: <%= prefix %>/projects/*&
Disallow: <%= prefix %>/projects/*%
<% end %>

# Don't crawl sorts or most queries.
# They have the same data, and crawling all the variants will produce
# slow crawl times.
{'fraction_non_alphanumeric': 0.12243767313019391, 'fraction_numerical': 0.0029547553093259462, 'mean_word_length': 4.515274949083503, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2477283', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1718, 'n_words': 642}