<commit_before># Blacklisted IPs (for a lot of requests)
#
# How to renew this list:
# /opt/google/appengine/appcfg.py --num_days=1 request_logs . gae.log
# awk '{ print $1 }' gae.log | sort | uniq -c | sort -rn | head -n 20 | egrep -v "$(awk '/- subnet/ { printf "|%s", $3 }' dos.yaml | sed 's/^|//')" | while read number ip; do printf "%5d %15s %35s %s\n" "$number" "$ip" "$(whois $ip | awk -F: '{ if($1 == "OrgName") { printf("%s", $2); exit 0; } }' | sed -r 's/^ +//g')" "$(awk -F '"' "/^$ip/ { printf(\"%s\", \$4); exit 0; }" gae.log)"; done

blacklist:
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.3.11
- subnet: 172.16.58.3
- subnet: 192.168.127.12
- subnet: 172.16.17.32
- subnet: 172.16.17.32
- subnet: 192.168.3.11/17
- subnet: 172.16.31.10/18
- subnet: 192.168.3.11/17
- subnet: 172.16.17.32
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.3.11
- subnet: 172.16.31.10<commit_msg>Update dos.yaml<commit_after># Blacklisted IPs (for a lot of requests)
#
# How to renew this list:
# /opt/google/appengine/appcfg.py --num_days=1 request_logs . gae.log
# awk '{ print $1 }' gae.log | sort | uniq -c | sort -rn | head -n 20 | egrep -v "$(awk '/- subnet/ { printf "|%s", $3 }' dos.yaml | sed 's/^|//')" | while read number ip; do printf "%5d %15s %35s %s\n" "$number" "$ip" "$(whois $ip | awk -F: '{ if($1 == "OrgName") { printf("%s", $2); exit 0; } }' | sed -r 's/^ +//g')" "$(awk -F '"' "/^$ip/ { printf(\"%s\", \$4); exit 0; }" gae.log)"; done

blacklist:
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.3.11
- subnet: 192.168.3.11
- subnet: 172.16.58.3
- subnet: 192.168.127.12
- subnet: 172.16.17.32
- subnet: 172.16.17.32
- subnet: 192.168.3.11/17
- subnet: 172.16.31.10/18
- subnet: 192.168.3.11/17
- subnet: 172.16.17.32
- subnet: 192.168.127.12
- subnet: 192.168.127.12
- subnet: 192.168.3.11
- subnet: 172.16.31.10
{'fraction_non_alphanumeric': 0.2431736218444101, 'fraction_numerical': 0.1859866048428645, 'mean_word_length': 2.79296875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 33, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5224799', 'n_tokens_mistral': 1128, 'n_tokens_neox': 881, 'n_words': 216}