<commit_before># for an installation in a subdirectory, you have to copy this file in your root of your domain and add  /subdir/ on each line
User-agent: *
Crawl-Delay: 30
Disallow: /CVS/
Disallow: /lib/
Disallow: /doc/
Disallow: /dump/
Disallow: /img/
Disallow: /templates/
Disallow: /templates_c/
Disallow: /var/
Disallow: /lang/
Disallow: /backups/
Disallow: /bin/
Disallow: /db/
Disallow: /images/
Disallow: /maps/
Disallow: /modules/
Disallow: /popups/
Disallow: /temp/
Disallow: /tests/
Disallow: /tikimovies/
Disallow: tiki-install.php
Disallow: /tiki-install.php
Disallow: tiki-admin.php
Disallow: /tiki-admin.php
Disallow: tiki-admin
Disallow: /tiki-admin
Disallow: tiki-editpage.php
Disallow: /tiki-editpage.php
Disallow: tiki-print.php
Disallow: /tiki-print.php
Disallow: tiki-print_article.php
Disallow: /tiki-print_article.php
Disallow: tiki-print_blog_post.php
Disallow: /tiki-print_blog_post.php
Disallow: tiki-print_multi_pages.php
Disallow: /tiki-print_multi_pages.php
Disallow: tiki-print_pages.php
Disallow: /tiki-print_pages.php
Disallow: tiki-pagehistory.php
Disallow: /tiki-pagehistory.php
Disallow: tiki-backlinks.php
Disallow: /tiki-backlinks.php
Disallow: tiki-likepages.php
Disallow: /tiki-likepages.php
Disallow: tiki-listpages.php
Disallow: /tiki-listpages.php
Disallow: tiki-lastchanges.php
Disallow: /tiki-lastchanges.php
Disallow: tiki-orphan_pages.php
Disallow: /tiki-orphan_pages.php
Disallow: tiki-register.php
Disallow: /tiki-register.php
Disallow: tiki-login_scr.php
Disallow: /tiki-login_scr.php
Disallow: tiki-view_cache.php
Disallow: /tiki-view_cache.php
Disallow: tiki-user_information.php<commit_msg>[ADD] robots.txt: attempt to reduce server overload by getting crawling robots lost in file galleries due to sort links (e.g. more than 7000 indexed variations of one Tiki filegals in Google for cca 10 galleries)<commit_after># for an installation in a subdirectory, you have to copy this file in your root of your domain and add  /subdir/ on each line
User-agent: *
Crawl-Delay: 30
Disallow: /CVS/
Disallow: /lib/
Disallow: /doc/
Disallow: /dump/
Disallow: /img/
Disallow: /templates/
Disallow: /templates_c/
Disallow: /var/
Disallow: /lang/
Disallow: /backups/
Disallow: /bin/
Disallow: /db/
Disallow: /images/
Disallow: /maps/
Disallow: /modules/
Disallow: /popups/
Disallow: /temp/
Disallow: /tests/
Disallow: /tikimovies/
Disallow: tiki-install.php
Disallow: /tiki-install.php
Disallow: tiki-admin.php
Disallow: /tiki-admin.php
Disallow: tiki-admin
Disallow: /tiki-admin
Disallow: tiki-editpage.php
Disallow: /tiki-editpage.php
Disallow: tiki-print.php
Disallow: /tiki-print.php
Disallow: tiki-print_article.php
Disallow: /tiki-print_article.php
Disallow: tiki-print_blog_post.php
Disallow: /tiki-print_blog_post.php
Disallow: tiki-print_multi_pages.php
Disallow: /tiki-print_multi_pages.php
Disallow: tiki-print_pages.php
Disallow: /tiki-print_pages.php
Disallow: tiki-pagehistory.php
Disallow: /tiki-pagehistory.php
Disallow: tiki-backlinks.php
Disallow: /tiki-backlinks.php
Disallow: tiki-likepages.php
Disallow: /tiki-likepages.php
Disallow: tiki-listpages.php
Disallow: /tiki-listpages.php
Disallow: tiki-list_file_gallery.php
Disallow: /tiki-list_file_gallery.php
Disallow: tiki-lastchanges.php
Disallow: /tiki-lastchanges.php
Disallow: tiki-orphan_pages.php
Disallow: /tiki-orphan_pages.php
Disallow: tiki-register.php
Disallow: /tiki-register.php
Disallow: tiki-login_scr.php
Disallow: /tiki-login_scr.php
Disallow: tiki-view_cache.php
Disallow: /tiki-view_cache.php
Disallow: tiki-user_information.php
{'fraction_non_alphanumeric': 0.11445444319460067, 'fraction_numerical': 0.00281214848143982, 'mean_word_length': 4.998313659359191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3157432', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1334, 'n_words': 317}