<commit_before># See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file

User-agent: *
Disallow: /account/
Disallow: /comment/add_comment/
Disallow: /comment/destroy_comment/
Disallow: /comment/edit_comment/
Disallow: /image/add_image/
Disallow: /image/add_image_to_obs/
Disallow: /image/advanced_obj_search
Disallow: /image/destroy_image/
Disallow: /image/edit_image/
Disallow: /image/image_search/
Disallow: /image/license_updater/
Disallow: /image/next_image/
Disallow: /image/prev_image/
Disallow: /image/remove_image/
Disallow: /image/reuse_image/
Disallow: /image/reuse_image_by_id/
Disallow: /image/reuse_image_for_user/
Disallow: /image/set_image_quality/
Disallow: /location/add_to_location/
Disallow: /location/auto_complete_location/
Disallow: /location/create_location/
Disallow: /location/edit_location/
Disallow: /location/list_merge_options
Disallow: /location/list_merge_options/
Disallow: /location/map_locations
Disallow: /location/map_locations/
Disallow: /location/where_search/
Disallow: /name/advanced_obj_search
Disallow: /name/approve_name/
Disallow: /name/author_request/
Disallow: /name/auto_complete_name
Disallow: /name/bulk_name_edit/
Disallow: /name/change_synonyms/
Disallow: /name/create_name/
Disallow: /name/deprecate_name/
Disallow: /name/edit_name/
Disallow: /name/email_tracking/
Disallow: /name/eol
Disallow: /name/eol_preview
Disallow: /name/name_search/
Disallow: /name/review_authors/
Disallow: /observer/add_comment/
Disallow: /observer/advanced_obj_search
Disallow: /observer/advanced_search
Disallow: /observer/advanced_search_results
Disallow: /observer/approve_name/
Disallow: /observer/ask_observation_question/
Disallow: /observer/ask_user_question/
Disallow: /observer/ask_webmaster_question
Disallow: /observer/change_synonyms/
Disallow: /observer/commercial_inquiry/
Disallow: /observer/create_naming/
Disallow: /observer/create_observation
Disallow: /observer/create_species_list/
Disallow: /observer/deprecate_name/
Disallow: /observer/destroy_naming/
Disallow: /observer/destroy_observation/
Disallow: /observer/edit_name/
Disallow: /observer/edit_naming/
Disallow: /observer/edit_observation/
Disallow: /observer/email_features
Disallow: /observer/license_updater/
Disallow: /observer/list_images/
Disallow: /observer/location_search
Disallow: /observer/location_search/
Disallow: /observer/manage_species_lists/
Disallow: /observer/next_observation/
Disallow: /observer/observation_search
Disallow: /observer/observation_search/
Disallow: /observer/observations_by_name
Disallow: /observer/observations_by_name/
Disallow: /observer/pattern_search/
Disallow: /observer/prev_observation/
Disallow: /observer/recalc/
Disallow: /observer/show_image/
Disallow: /observer/show_name/
Disallow: /observer/show_past_name/
Disallow: /observer/show_species_list/
Disallow: /observer/textile_sandbox
Disallow: /observer/users_by_name/
Disallow: /project/show_draft/
Disallow: /species_list/add_observation_to_species_list/
Disallow: /species_list/create_species_list/
Disallow: /species_list/edit_species_list/
Disallow: /species_list/make_report/
Disallow: /species_list/manage_species_lists/
Disallow: /species_list/upload_species_list/<commit_msg>Added a few expensive pages to robots.txt disallow list.<commit_after># See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file

User-agent: *
Disallow: /account/
Disallow: /comment/add_comment/
Disallow: /comment/destroy_comment/
Disallow: /comment/edit_comment/
Disallow: /image/add_image/
Disallow: /image/add_image_to_obs/
Disallow: /image/advanced_obj_search
Disallow: /image/destroy_image/
Disallow: /image/edit_image/
Disallow: /image/image_search/
Disallow: /image/license_updater/
Disallow: /image/next_image/
Disallow: /image/prev_image/
Disallow: /image/remove_image/
Disallow: /image/reuse_image/
Disallow: /image/reuse_image_by_id/
Disallow: /image/reuse_image_for_user/
Disallow: /image/set_image_quality/
Disallow: /image/show_original/
Disallow: /location/add_to_location/
Disallow: /location/auto_complete_location/
Disallow: /location/create_location/
Disallow: /location/edit_location/
Disallow: /location/list_merge_options
Disallow: /location/list_merge_options/
Disallow: /location/map_locations
Disallow: /location/map_locations/
Disallow: /location/where_search/
Disallow: /name/advanced_obj_search
Disallow: /name/approve_name/
Disallow: /name/author_request/
Disallow: /name/auto_complete_name
Disallow: /name/bulk_name_edit/
Disallow: /name/change_synonyms/
Disallow: /name/create_name/
Disallow: /name/deprecate_name/
Disallow: /name/edit_name/
Disallow: /name/email_tracking/
Disallow: /name/eol
Disallow: /name/eol_preview
Disallow: /name/name_index/
Disallow: /name/name_search/
Disallow: /name/review_authors/
Disallow: /observer/add_comment/
Disallow: /observer/advanced_obj_search
Disallow: /observer/advanced_search
Disallow: /observer/advanced_search_results
Disallow: /observer/approve_name/
Disallow: /observer/ask_observation_question/
Disallow: /observer/ask_user_question/
Disallow: /observer/ask_webmaster_question
Disallow: /observer/change_synonyms/
Disallow: /observer/commercial_inquiry/
Disallow: /observer/create_naming/
Disallow: /observer/create_observation
Disallow: /observer/create_species_list/
Disallow: /observer/deprecate_name/
Disallow: /observer/destroy_naming/
Disallow: /observer/destroy_observation/
Disallow: /observer/edit_name/
Disallow: /observer/edit_naming/
Disallow: /observer/edit_observation/
Disallow: /observer/email_features
Disallow: /observer/license_updater/
Disallow: /observer/list_images/
Disallow: /observer/location_search
Disallow: /observer/location_search/
Disallow: /observer/manage_species_lists/
Disallow: /observer/next_observation/
Disallow: /observer/observation_search
Disallow: /observer/observation_search/
Disallow: /observer/observations_by_name
Disallow: /observer/observations_by_name/
Disallow: /observer/pattern_search/
Disallow: /observer/prev_observation/
Disallow: /observer/recalc/
Disallow: /observer/show_image/
Disallow: /observer/show_name/
Disallow: /observer/show_past_name/
Disallow: /observer/show_site_stats
Disallow: /observer/show_species_list/
Disallow: /observer/textile_sandbox
Disallow: /observer/users_by_name/
Disallow: /project/show_draft/
Disallow: /species_list/add_observation_to_species_list/
Disallow: /species_list/create_species_list/
Disallow: /species_list/edit_species_list/
Disallow: /species_list/make_report/
Disallow: /species_list/manage_species_lists/
Disallow: /species_list/name_lister
Disallow: /species_list/species_list_by_title
Disallow: /species_list/upload_species_list/
{'fraction_non_alphanumeric': 0.10619865571321882, 'fraction_numerical': 0.0, 'mean_word_length': 6.334063526834611, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64361', 'n_tokens_mistral': 2406, 'n_tokens_neox': 2270, 'n_words': 391}