<commit_before>User-agent: *
Disallow: /login$
Disallow: /connexion$
Disallow: /request
Disallow: /api*
Disallow: /*/new$
Disallow: /*/create*$
Disallow: /*/edit$
Disallow: /*/delete$
Disallow: /*/update$
Disallow: /rechercher/*
Disallow: /*download$
Disallow: /*.xhr$
Disallow: /*.geojson$
Disallow: /*?referer=1$
Disallow: /*?q=*

User-agent: Googlebot-Image
Allow: /media/

Sitemap: https://www.lairdubois.fr/sitemap-index.xml
<commit_msg>~ robots.txt update<commit_after>User-agent: *
Disallow: /login$
Disallow: /connexion$
Disallow: /request
Disallow: /api*
Disallow: /*/new$
Disallow: /*/create*$
Disallow: /*/edit$
Disallow: /*/delete$
Disallow: /*/update$
Disallow: /*/lock$
Disallow: /*/unlock$
Disallow: /rechercher/*
Disallow: /likes/*
Disallow: /votes/*
Disallow: /*download$
Disallow: /*.xhr$
Disallow: /*.geojson$
Disallow: /*?referer=1$
Disallow: /*?q=*

User-agent: Googlebot-Image
Allow: /media/

Sitemap: https://www.lairdubois.fr/sitemap-index.xml

{'fraction_non_alphanumeric': 0.2024793388429752, 'fraction_numerical': 0.002066115702479339, 'mean_word_length': 4.7, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5674403', 'n_tokens_mistral': 396, 'n_tokens_neox': 394, 'n_words': 84}