<commit_before>User-agent: *
Disallow: /bills/tags
Disallow: /search
Disallow: /charts
Disallow: /process-tags.php
Disallow: /process-comments.php
Disallow: /process-polls.php
Disallow: /process-pingbacks.php
Disallow: /downloads/bills/

User-agent: Speedy
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: BecomeBot 
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Sproose
Disallow: /

User-agent: WebAlta Crawler
Disallow: /

User-agent: SummizeBot
Disallow: /

User-agent: wwwster
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: BlogPulseLive
Disallow: /

User-agent: Sphere Scout
Disallow: /

User-agent: IIITBOT
Disallow: /

User-agent: BDFetch
Disallow: /

User-agent: Snapbot
Disallow: /

User-agent: dotbot 
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Accelobot
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: NG
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: msnbot <commit_msg>Block annoying bots<commit_after>User-agent: *
Disallow: /bills/tags
Disallow: /search
Disallow: /charts
Disallow: /process-tags.php
Disallow: /process-comments.php
Disallow: /process-polls.php
Disallow: /downloads/bills/

User-agent: Speedy
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: BecomeBot 
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Sproose
Disallow: /

User-agent: WebAlta Crawler
Disallow: /

User-agent: SummizeBot
Disallow: /

User-agent: wwwster
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: BlogPulseLive
Disallow: /

User-agent: Sphere Scout
Disallow: /

User-agent: IIITBOT
Disallow: /

User-agent: BDFetch
Disallow: /

User-agent: Snapbot
Disallow: /

User-agent: dotbot 
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Accelobot
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: NG
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: msnbot 
{'fraction_non_alphanumeric': 0.1205607476635514, 'fraction_numerical': 0.001869158878504673, 'mean_word_length': 4.025821596244131, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63736', 'n_tokens_mistral': 880, 'n_tokens_neox': 860, 'n_words': 186}