<commit_before># See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#        _
#       ( )
#        H
#        H
#       _H_
#    .-'-.-'-.
#   /         \
#  |           |
#  |   .-------'._
#  |  / /  '.' '. \
#  |  \ \ @   @ / /
#  |   '---------'
#  |    _______|
#  |  .'-+-+-+|
#  |  '.-+-+-+|  ------ KILL ALL HUMANS!
#  |    """""" |
#  '-.__   __.-'
#       """  sul
#
#   Art by <NAME>
#
#
#! User-agent: Yandex
#! Disallow: /assets/
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /*.xls$
#! Allow: /
#! Host: example.com
#! 
#! User-agent: Googlebot
#! Disallow: /assets/
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /*.xls$
#! Allow: /
#! 
#! User-agent: *
#! Disallow: /assets/
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /temp/privatedata.xls
#! Disallow:
#! 
#! Sitemap: http://example.com/sitemap.xml.gz

# Ban all spiders from the entire site
User-agent: *
Disallow: /
<commit_msg>Allow robots crawle assets by default according rules of Google<commit_after># See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#        _
#       ( )
#        H
#        H
#       _H_
#    .-'-.-'-.
#   /         \
#  |           |
#  |   .-------'._
#  |  / /  '.' '. \
#  |  \ \ @   @ / /
#  |   '---------'
#  |    _______|
#  |  .'-+-+-+|
#  |  '.-+-+-+|  ------ KILL ALL HUMANS!
#  |    """""" |
#  '-.__   __.-'
#       """  sul
#
#   Art by <NAME>
#
#
#! User-agent: Yandex
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /*.xls$
#! Allow: /
#! Host: example.com
#! 
#! User-agent: Googlebot
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /*.xls$
#! Allow: /
#! 
#! User-agent: *
#! Disallow: /admin/
#! Disallow: /humans.txt
#! Disallow: /temp/privatedata.xls
#! Disallow:
#! 
#! Sitemap: http://example.com/sitemap.xml.gz

# Ban all spiders from the entire site
User-agent: *
Disallow: /

{'fraction_non_alphanumeric': 0.2494877049180328, 'fraction_numerical': 0.0, 'mean_word_length': 1.7702127659574467, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180533', 'n_tokens_mistral': 857, 'n_tokens_neox': 837, 'n_words': 137}