<commit_before>User-agent: *
Crawl-delay: 5
Sitemap: https://developer.mozilla.org/sitemap.xml
Request-rate: 1/5

Disallow: /*admin*
Disallow: /*docs/feeds
Disallow: /*docs/templates
Disallow: /*docs*Template:
Disallow: /*dashboards*
Disallow: /*docs/all
Disallow: /*docs/tag*
Disallow: /*docs/needs-review*
Disallow: /*docs/localization-tag*
Disallow: /*docs/with-errors
Disallow: /*docs/without-parent
Disallow: /*docs/top-level
Disallow: /*docs/new
Disallow: /*docs/get-documents
Disallow: /*docs/submit_akismet_spam
Disallow: /*docs/load*
Disallow: /*docs*$compare
Disallow: /*docs*$revision
Disallow: /*docs*$history
Disallow: /*docs*$edit
Disallow: /*docs*$children
Disallow: /*docs*$flag
Disallow: /*docs*$translate
Disallow: /*docs*$locales
Disallow: /*docs*$json
Disallow: /*docs*$styles
Disallow: /*docs*$toc
Disallow: /*docs*$move
Disallow: /*docs*$quick-review
Disallow: /*docs*$samples
Disallow: /*docs*$revert
Disallow: /*docs*$repair_breadcrumbs
Disallow: /*docs*$delete
Disallow: /*docs*$restore
Disallow: /*docs*$purge
Disallow: /*docs*$subscribe
Disallow: /*docs*$subscribe_to_tree
Disallow: /*docs*$vote
Disallow: /*docs.json
Disallow: /*docs/ckeditor_config.js
Disallow: /*files/
Disallow: /media
Disallow: /*move-requested
Disallow: /*preview-wiki-content
Disallow: /*profiles*edit
Disallow: /*search*
Disallow: /skins
Disallow: /*type=feed
Disallow: /*users/<commit_msg>bug 1281270 - robots.txt edits: Part II (#3891)<commit_after>User-agent: *
Crawl-delay: 5
Sitemap: https://developer.mozilla.org/sitemap.xml
Request-rate: 1/5

Disallow: /admin/
Disallow: /*/dashboards/*
Disallow: /*docs/feeds
Disallow: /*docs/templates
Disallow: /*docs*Template:
Disallow: /*docs/all
Disallow: /*docs/tag*
Disallow: /*docs/needs-review*
Disallow: /*docs/localization-tag*
Disallow: /*docs/with-errors
Disallow: /*docs/without-parent
Disallow: /*docs/top-level
Disallow: /*docs/new
Disallow: /*docs/get-documents
Disallow: /*docs/submit_akismet_spam
Disallow: /*docs/load*
Disallow: /*$compare
Disallow: /*$revision
Disallow: /*$history
Disallow: /*$edit
Disallow: /*$children
Disallow: /*$flag
Disallow: /*$translate
Disallow: /*$locales
Disallow: /*$json
Disallow: /*$styles
Disallow: /*$toc
Disallow: /*$move
Disallow: /*$quick-review
Disallow: /*$samples
Disallow: /*$revert
Disallow: /*$repair_breadcrumbs
Disallow: /*$delete
Disallow: /*$restore
Disallow: /*$purge
Disallow: /*$subscribe
Disallow: /*$subscribe_to_tree
Disallow: /*$vote
Disallow: /*docs.json
Disallow: /*docs/ckeditor_config.js
Disallow: /*/files/
Disallow: /media
Disallow: /*move-requested
Disallow: /*preview-wiki-content
Disallow: /*profiles*/edit
Disallow: /skins
Disallow: /*type=feed
Disallow: /*users/
{'fraction_non_alphanumeric': 0.17725752508361203, 'fraction_numerical': 0.0063173541434411, 'mean_word_length': 5.763819095477387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6389489', 'n_tokens_mistral': 1001, 'n_tokens_neox': 974, 'n_words': 213}