<commit_before># The (extremely complex) rules for domain delegation.
# Note that really, this should be called "RegistryBoundaries"; see bug 4605

# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>

=head1 NAME

Mail::SpamAssassin::Util::RegistrarBoundaries - domain delegation rules

=cut

package Mail::SpamAssassin::Util::RegistrarBoundaries;

use strict;
use warnings;
use bytes;
use re 'taint';

use vars qw (
  @ISA %TWO_LEVEL_DOMAINS %THREE_LEVEL_DOMAINS %US_STATES %VALID_TLDS $VALID_TLDS_RE
);

# %VALID_TLDS
# The list of currently-valid TLDs for the DNS system.
# Note for SA devs only: If required, when updating domain lists, also modify t/uri_text.t accordingly
#
# bash line to generate a formatted list of domains
# Fetches domains, drops the top comment line, then joins domains with spaces in between
#   wget http://data.iana.org/TLD/tlds-alpha-by-domain.txt -O - | tail -n+2 | perl -e 'chomp && s/$/ / && print lc while <>' && echo
#
# Remember to also change regexp below when updating!
#
# Last update: 2014-11-16-axb
foreach (qw/
abogado ac academy accountants active actor ad ae aero af ag agency ai airforce al allfinanz alsace am an android ao aq ar archi army arpa as asia associates at attorney au auction audio autos aw ax axa az ba band bar bargains bayern bb bd be beer berlin best bf bg bh bi bid bike bio biz bj black blackfriday bloomberg blue bm bmw bn bnpparibas bo boo boutique br brussels bs bt budapest build builders business buzz bv bw by bz bzh ca cab cal camera camp cancerresearch capetown capital caravan cards care career careers casa cash cat catering cc cd center ceo cern cf cg ch channel cheap christmas chrome church ci citic city ck cl claims cleaning click clinic clothing club cm cn co codes coffee college cologne com community company computer condos construction consulting contractors cooking cool coop country cr credit creditcard crs cruises cu cuisinella cv cw cx cy cymru cz dad dance dating day de deals degree delivery democrat dental dentist desi diamonds diet digital direct directory discount dj dk dm dnp do domains durban dvag dz eat ec edu education ee eg email emerck energy engineer engineering enterprises equipment er es esq estate et eu eus events exchange expert exposed fail farm feedback fi finance financial fish fishing fitness fj fk flights florist flsmidth fly fm fo foo forsale foundation fr frl frogans fund furniture futbol ga gal gallery gb gbiz gd ge gent gf gg gh gi gift gifts gives gl glass gle global globo gm gmail gmo gmx gn google gop gov gp gq gr graphics gratis green gripe gs gt gu guide guitars guru gw gy hamburg haus healthcare help here hiphop hiv hk hm hn holdings holiday homes horse host hosting house how hr ht hu ibm id ie il im immo immobilien in industries info ing ink institute insure int international investments io iq ir is it je jetzt jm jo jobs joburg jp juegos kaufen ke kg kh ki kim kitchen kiwi km kn koeln kp kr krd kred kw ky kz la lacaixa land lawyer lb lc lease lgbt li life lighting limited limo link lk loans london lotto lr ls lt ltda lu luxe luxury lv ly ma maison management mango market marketing mc md me media meet melbourne meme menu mg mh miami mil mini mk ml mm mn mo mobi moda moe monash mortgage moscow motorcycles mov mp mq mr ms mt mu museum mv mw mx my mz na nagoya name navy nc ne net network neustar new nexus nf ng ngo nhk ni ninja nl no np nr nra nrw nu nyc nz okinawa om ong onl ooo org organic otsuka ovh pa paris partners parts pe pf pg ph pharmacy photo photography photos physio pics pictures pink pizza pk pl place plumbing pm pn pohl poker post pr praxi press pro prod productions prof properties property ps pt pub pw py qa qpon quebec re realtor recipes red rehab reise reisen reit ren rentals repair report republican rest restaurant reviews rich rio rip ro rocks rodeo rs rsvp ru ruhr rw ryukyu sa saarland sarl sb sc sca scb schmidt schule science scot sd se services sexy sg sh shiksha shoes si singles sj sk sl sm sn so social software sohu solar solutions soy space spiegel sr st su supplies supply support surf surgery suzuki sv sx sy sydney systems sz taipei tatar tattoo tax tc td technology tel tf tg th tienda tips tirol tj tk tl tm tn to today tokyo tools top town toys tp tr trade training travel tt tui tv tw tz ua ug uk university uno uol us uy uz va vacations vc ve vegas ventures versicherung vet vg vi viajes villas vision vlaanderen vn vodka vote voting voto voyage vu wales wang watch webcam website wed wedding wf whoswho wien wiki williamhill wme work works world ws wtc wtf xn--1qqw23a xn--3bst00m xn--3ds443g xn--3e0b707e xn--45brj9c xn--4gbrim xn--55qw42g xn--55qx5d xn--6frz82g xn--6qq986b3xl xn--80adxhks xn--80ao21a xn--80asehdb xn--80aswg xn--90a3ac xn--c1avg xn--cg4bki xn--clchc0ea0b2g2a9gcd xn--czr694b xn--czru2d xn--d1acj3b xn--d1alf xn--fiq228c5hs xn--fiq64b xn--fiqs8s xn--fiqz9s xn--fpcrj9c3d xn--fzc2c9e2c xn--gecrj9c xn--h2brj9c xn--i1b6b1a6a2e xn--io0a7i xn--j1amh xn--j6w193g xn--kprw13d xn--kpry57d xn--kput3i xn--l1acc xn--lgbbat1ad8j xn--mgb9awbf xn--mgba3a4f16a xn--mgbaam7a8h xn--mgbab2bd xn--mgbayh7gpa xn--mgbbh1a71e xn--mgbc0a9azcg xn--mgberp4a5d4ar xn--mgbx4cd0ab xn--ngbc5azd xn--node xn--nqv7f xn--nqv7fs00ema xn--o3cw4h xn--ogbpf8fl xn--p1acf xn--p1ai xn--pgbs0dh xn--q9jyb4c xn--rhqv96g xn--s9brj9c xn--ses554g xn--unup4y xn--vermgensberater-ctb xn--vermgensberatung-pwb xn--vhquv xn--wgbh1c xn--wgbl6a xn--xhq521b xn--xkc2al3hye2a xn--xkc2dl3a5ee0h xn--yfro4i67o xn--ygbi2ammx xn--zfr164b xxx xyz yachts yandex ye yoga yokohama youtube yt za zip zm zone zw
  /) {
  $VALID_TLDS{$_} = 1;
}

# $VALID_TLDS_RE
# %VALID_TLDS as Regexp::List optimized regexp, for use in Plugins etc
# bash line to generate regex from TLD list
# Fetches domains, drops the top commet line, builds a regex from the list of domains, then formats it to remove (?-xsim:) regex modifier flags
#   wget http://data.iana.org/TLD/tlds-alpha-by-domain.txt -O - | tail -n+2 | perl -MRegexp::List -e '$/=undef; $_=<>; $r = Regexp::List->new; push @l, $_ for (split); print $r->list2re(@l)' | perl -pe 's/^\(\?[^:]*:(.*)\)$/$1/' && echo
#
$VALID_TLDS_RE = qr/
(?:X(?:N--(?:MGB(?:A(?:(?:3A4F16|YH7GP)A|AM7A8H|B2BD)|ERP4A5D4AR|C0A9AZCG|BH1A71E|X4CD0AB|9AWBF)|F(?:IQ(?:(?:228C5H|S8|Z9)S|64B)|PCRJ9C3D|ZC2C9E2C)|C(?:LCHC0EA0B2G2A9GCD|ZR(?:694B|U2D)|G4BKI|1AVG)|V(?:ERMGENSBERAT(?:UNG-PW|ER-CT)B|HQUV)|X(?:KC2(?:DL3A5EE0H|AL3HYE2A)|HQ521B)|(?:(?:GEC|H2B)RJ9|Q9JYB4|90A3A)C|N(?:QV7F(?:S00EMA)?|GBC5AZD|ODE)|80A(?:S(?:EHDB|WG)|DXHKS|O21A)|3(?:E0B707E|BST00M|DS443G)|KP(?:R(?:W13|Y57)D|UT3I)|P(?:1A(?:CF|I)|GBS0DH)|Y(?:FRO4I67O|GBI2AMMX)|6(?:QQ986B3XL|FRZ82G)|I(?:1B6B1A6A2E|O0A7I)|L(?:GBBAT1AD8J|1ACC)|O(?:GBPF8FL|3CW4H)|S(?:9BRJ9C|ES554G)|4(?:5BRJ9C|GBRIM)|J(?:6W193G|1AMH)|55Q(?:W42G|X5D)|D1A(?:CJ3B|LF)|WGB(?:H1C|L6A)|1QQW23A|RHQV96G|ZFR164B|UNUP4Y)|XX|YZ)|C(?:[CDFGKMNVWXZ]|O(?:N(?:S(?:TRUCTION|ULTING)|(?:TRACTOR|DO)S)|M(?:P(?:UTER|ANY)|MUNITY)?|(?:L(?:LEG|OGN)|FFE)E|O(?:[LP]|KING)|UNTRY|DES)?|A(?:[BL]|R(?:E(?:ERS?)?|AVAN|DS)|P(?:ETOWN|ITAL)|NCERRESEARCH|T(?:ERING)?|M(?:ERA|P)|S[AH])?|L(?:(?:EAN|OTH)ING|I(?:NIC|CK)|AIMS|UB)?|H(?:R(?:ISTMAS|OME)|ANNEL|URCH|EAP)?|R(?:EDIT(?:CARD)?|(?:UISE)?S)?|I(?:T(?:IC|Y))?|E(?:NTER|RN|O)|U(?:ISINELLA)?|Y(?:MRU)?)|S(?:[BDGJKLMNRTVXZ]|U(?:PP(?:L(?:IES|Y)|ORT)|R(?:GERY|F)|ZUKI)?|O(?:L(?:UTIONS|AR)|FTWARE|CIAL|HU|Y)?|C(?:[AB]|H(?:MIDT|ULE)|IENCE|OT)?|Y(?:STEMS|DNEY)?|A(?:ARLAND|RL)?|E(?:RVICES|XY)?|H(?:IKSHA|OES)?|P(?:IEGEL|ACE)|I(?:NGLES)?)|A(?:[DFMQWZ]|C(?:T(?:IVE|OR)|COUNTANTS|ADEMY)?|U(?:CTION|DIO|TOS)?|L(?:LFINANZ|SACE)?|S(?:SOCIATES|IA)?|R(?:CHI|MY|PA)?|I(?:RFORCE)?|T(?:TORNEY)?|(?:BOGAD)?O|N(?:DROID)?|G(?:ENCY)?|E(?:RO)?|XA?)|B(?:[BDFGHJSTVWY]|U(?:ILD(?:ERS)?|DAPEST|SINESS|ZZ)|L(?:ACK(?:FRIDAY)?|OOMBERG|UE)|A(?:R(?:GAINS)?|YERN|ND)?|E(?:RLIN|ER|ST)?|I(?:[DOZ]|KE)?|N(?:PPARIBAS)?|O(?:UTIQUE|O)?|R(?:USSELS)?|MW?|ZH?)|P(?:[EFGKMNSTWY]|R(?:O(?:D(?:UCTIONS)?|PERT(?:IES|Y)|F)?|AXI|ESS)?|H(?:OTO(?:GRAPHY|S)?|ARMACY|YSIO)?|A(?:R(?:T(?:NER)?|I)S)?|I(?:C(?:TURE)?S|ZZA|NK)|L(?:UMBING|ACE)?|O(?:KER|HL|ST)|UB)|G(?:[DFGHNPQSTWY]|R(?:A(?:PHIC|TI)S|EEN|IPE)?|L(?:OB(?:AL|O)|ASS|E)?|U(?:I(?:TARS|DE)|RU)?|A(?:L(?:LERY)?)?|I(?:FTS?|VES)?|M(?:[OX]|AIL)?|O(?:[PV]|OGLE)|B(?:IZ)?|E(?:NT)?)|M(?:[CDGHKLMNPQRSTVWXYZ]|O(?:(?:RTGAG)?E|TORCYCLES|NASH|SCOW|BI|DA|V)?|A(?:N(?:AGEMENT|GO)|RKET(?:ING)?|ISON)?|E(?:(?:LBOURN|M)E|DIA|ET|NU)?|I(?:(?:AM|N)I|L)|U(?:SEUM)?)|F(?:[JM]|I(?:NANC(?:IAL|E)|SH(?:ING)?|TNESS)?|L(?:SMIDTH|IGHTS|ORIST|Y)|O(?:UNDATION|RSALE|O)?|U(?:RNITURE|TBOL|ND)|R(?:OGANS|L)?|(?:EEDBAC)?K|A(?:IL|RM))|R(?:E(?:P(?:UBLICAN|AIR|ORT)|(?:CIPE|VIEW)S|S(?:TAURAN)?T|I(?:SEN?|T)|N(?:TALS)?|ALTOR|HAB|D)?|O(?:CKS|DEO)?|I(?:[OP]|CH)|S(?:VP)?|U(?:HR)?|YUKYU|W)|D(?:[JKMZ]|E(?:NT(?:IST|AL)|LIVERY|MOCRAT|GREE|ALS|SI)?|I(?:(?:SCOUN|E)T|RECT(?:ORY)?|AMONDS|GITAL)|A(?:[DY]|TING|NCE)|O(?:MAINS)?|URBAN|VAG|NP)|E(?:[CEGR]|N(?:GINEER(?:ING)?|TERPRISES|ERGY)|X(?:P(?:OSED|ERT)|CHANGE)|(?:QUIPMEN|A)?T|DU(?:CATION)?|M(?:ERCK|AIL)|S(?:TATE|Q)?|VENTS|US?)|T(?:[CDFGHJKLMNPTVWZ]|O(?:(?:OL|Y)S|DAY|KYO|WN|P)?|A(?:T(?:TOO|AR)|IPEI|X)|R(?:A(?:INING|VEL|DE))?|I(?:ENDA|ROL|PS)|E(?:CHNOLOGY|L)|UI)|V(?:[CGU]|E(?:(?:NTURE|GA)S|RSICHERUNG|T)?|O(?:T(?:[EO]|ING)|YAGE|DKA)|I(?:(?:AJE|LLA)S|SION)?|(?:LAANDERE)?N|A(?:CATIONS)?)|L(?:[BCKRSVY]|I(?:M(?:ITED|O)|GHTING|FE|NK)?|A(?:CAIXA|WYER|ND)?|O(?:NDON|ANS|TTO)|U(?:X(?:URY|E))?|T(?:DA)?|EASE|GBT)|H(?:[KMNRTU]|O(?:L(?:DINGS|IDAY)|ST(?:ING)?|[RU]SE|MES|W)|E(?:(?:ALTHCA)?RE|LP)|A(?:MBURG|US)|I(?:PHOP|V))|I(?:[DELOQRST]|N(?:[GK]|(?:VESTMENT|DUSTRIE)S|T(?:ERNATIONAL)?|S(?:TITUT|UR)E|FO)?|M(?:MO(?:BILIEN)?)?|BM)|W(?:[FS]|E(?:B(?:SITE|CAM)|D(?:DING)?)|I(?:LLIAMHILL|EN|KI)|A(?:LES|TCH|NG)|OR(?:KS?|LD)|HOSWHO|T[CF]|ME)|N(?:[FLOPUZ]|E(?:T(?:WORK)?|USTAR|XUS|W)?|A(?:GOYA|ME|VY)?|I(?:NJA)?|R[AW]?|GO?|Y?C|HK)|K(?:[EGHMPWYZ]|I(?:TCHEN|WI|M)?|(?:AUFE|OEL)?N|R(?:E?D)?)|Y(?:[ET]|O(?:(?:KOHAM|G)A|UTUBE)|A(?:CHTS|NDEX))|O(?:(?:KINAW|TSUK)A|RG(?:ANIC)?|N[GL]|OO|VH|M)|J(?:[MP]|O(?:B(?:URG|S))?|E(?:TZT)?|UEGOS)|U(?:[AGKSYZ]|N(?:IVERSITY|O)|OL)|Z(?:[AMW]|ONE|IP)|Q(?:UEBEC|PON|A))/ix;

# Two-Level TLDs
#
# to resort this, pump the whole list through:
#  perl -e '$/=undef; $_=<>; foreach(split) { ($a,$b) = split(/\./, $_, 2); $t{$b}->{$_}=1; } foreach (sort keys %t) { print "  ",join(" ", sort keys %{$t{$_}}),"\n" }'
#
# http://www.neustar.us/policies/docs/rfc_1480.txt
# data originally from http://spamcheck.freeapp.net/two-level-tlds
# The freeapp.net site now says that information on the site is obsolete
# See discussion and sources in comments of bug 5677
# updated as per bug 5815
# cleanup in progress per bug 6795 (axb)
# Unsorted sources:
# .ua : http://hostmaster.ua
# .hu : http://www.domain.hu/domain/English/szabalyzat/sld.html
#
foreach(qw/<commit_msg># Version 2014121701, Last Updated Thu Dec 18 07:07:01 2014 UTC<commit_after># The (extremely complex) rules for domain delegation.
# Note that really, this should be called "RegistryBoundaries"; see bug 4605

# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>

=head1 NAME

Mail::SpamAssassin::Util::RegistrarBoundaries - domain delegation rules

=cut

package Mail::SpamAssassin::Util::RegistrarBoundaries;

use strict;
use warnings;
use bytes;
use re 'taint';

use vars qw (
  @ISA %TWO_LEVEL_DOMAINS %THREE_LEVEL_DOMAINS %US_STATES %VALID_TLDS $VALID_TLDS_RE
);

# %VALID_TLDS
# The list of currently-valid TLDs for the DNS system.
# Note for SA devs only: If required, when updating domain lists, also modify t/uri_text.t accordingly
#
# bash line to generate a formatted list of domains
# Fetches domains, drops the top comment line, then joins domains with spaces in between
#   wget http://data.iana.org/TLD/tlds-alpha-by-domain.txt -O - | tail -n+2 | perl -e 'chomp && s/$/ / && print lc while <>' && echo
#
# Remember to also change regexp below when updating!
#
# Last update: 2014-12-17-axb
foreach (qw/
abogado ac academy accountants active actor ad adult ae aero af ag agency ai airforce al allfinanz alsace am an android ao aq aquarelle ar archi army arpa as asia associates at attorney au auction audio autos aw ax axa az ba band bar bargains bayern bb bd be beer berlin best bf bg bh bi bid bike bio biz bj black blackfriday bloomberg blue bm bmw bn bnpparibas bo boo boutique br brussels bs bt budapest build builders business buzz bv bw by bz bzh ca cab cal camera camp cancerresearch capetown capital caravan cards care career careers cartier casa cash cat catering cc cd center ceo cern cf cg ch channel cheap christmas chrome church ci citic city ck cl claims cleaning click clinic clothing club cm cn co coach codes coffee college cologne com community company computer condos construction consulting contractors cooking cool coop country cr credit creditcard cricket crs cruises cu cuisinella cv cw cx cy cymru cz dad dance dating day de deals degree delivery democrat dental dentist desi diamonds diet digital direct directory discount dj dk dm dnp do domains doosan durban dvag dz eat ec edu education ee eg email emerck energy engineer engineering enterprises equipment er es esq estate et eu eurovision eus events everbank exchange expert exposed fail farm fashion feedback fi finance financial firmdale fish fishing fitness fj fk flights florist flsmidth fly fm fo foo forsale foundation fr frl frogans fund furniture futbol ga gal gallery garden gb gbiz gd ge gent gf gg gh gi gift gifts gives gl glass gle global globo gm gmail gmo gmx gn google gop gov gp gq gr graphics gratis green gripe gs gt gu guide guitars guru gw gy hamburg haus healthcare help here hiphop hiv hk hm hn holdings holiday homes horse host hosting house how hr ht hu ibm id ie il im immo immobilien in industries info ing ink institute insure int international investments io iq ir irish is it iwc je jetzt jm jo jobs joburg jp juegos kaufen ke kg kh ki kim kitchen kiwi km kn koeln kp kr krd kred kw ky kz la lacaixa land latrobe lawyer lb lc lds lease legal lgbt li lidl life lighting limited limo link lk loans london lotto lr ls lt ltda lu luxe luxury lv ly ma madrid maison management mango market marketing mc md me media meet melbourne meme memorial menu mg mh miami mil mini mk ml mm mn mo mobi moda moe monash money mormon mortgage moscow motorcycles mov mp mq mr ms mt mu museum mv mw mx my mz na nagoya name navy nc ne net network neustar new nexus nf ng ngo nhk ni ninja nl no np nr nra nrw nu nyc nz okinawa om ong onl ooo org organic osaka otsuka ovh pa paris partners parts party pe pf pg ph pharmacy photo photography photos physio pics pictures pink pizza pk pl place plumbing pm pn pohl poker porn post pr praxi press pro prod productions prof properties property ps pt pub pw py qa qpon quebec re realtor recipes red rehab reise reisen reit ren rentals repair report republican rest restaurant reviews rich rio rip ro rocks rodeo rs rsvp ru ruhr rw ryukyu sa saarland samsung sarl sb sc sca scb schmidt schule schwarz science scot sd se services sew sexy sg sh shiksha shoes si singles sj sk sky sl sm sn so social software sohu solar solutions soy space spiegel sr st su supplies supply support surf surgery suzuki sv sx sy sydney systems sz taipei tatar tattoo tax tc td technology tel tf tg th tienda tips tirol tj tk tl tm tn to today tokyo tools top town toys tp tr trade training travel trust tt tui tv tw tz ua ug uk university uno uol us uy uz va vacations vc ve vegas ventures versicherung vet vg vi viajes villas vision vlaanderen vn vodka vote voting voto voyage vu wales wang watch webcam website wed wedding wf whoswho wien wiki williamhill wme work works world ws wtc wtf xn--1qqw23a xn--3bst00m xn--3ds443g xn--3e0b707e xn--45brj9c xn--45q11c xn--4gbrim xn--55qw42g xn--55qx5d xn--6frz82g xn--6qq986b3xl xn--80adxhks xn--80ao21a xn--80asehdb xn--80aswg xn--90a3ac xn--c1avg xn--cg4bki xn--clchc0ea0b2g2a9gcd xn--czr694b xn--czrs0t xn--czru2d xn--d1acj3b xn--d1alf xn--fiq228c5hs xn--fiq64b xn--fiqs8s xn--fiqz9s xn--flw351e xn--fpcrj9c3d xn--fzc2c9e2c xn--gecrj9c xn--h2brj9c xn--hxt814e xn--i1b6b1a6a2e xn--io0a7i xn--j1amh xn--j6w193g xn--kprw13d xn--kpry57d xn--kput3i xn--l1acc xn--lgbbat1ad8j xn--mgb9awbf xn--mgba3a4f16a xn--mgbaam7a8h xn--mgbab2bd xn--mgbayh7gpa xn--mgbbh1a71e xn--mgbc0a9azcg xn--mgberp4a5d4ar xn--mgbx4cd0ab xn--ngbc5azd xn--node xn--nqv7f xn--nqv7fs00ema xn--o3cw4h xn--ogbpf8fl xn--p1acf xn--p1ai xn--pgbs0dh xn--q9jyb4c xn--qcka1pmc xn--rhqv96g xn--s9brj9c xn--ses554g xn--unup4y xn--vermgensberater-ctb xn--vermgensberatung-pwb xn--vhquv xn--wgbh1c xn--wgbl6a xn--xhq521b xn--xkc2al3hye2a xn--xkc2dl3a5ee0h xn--yfro4i67o xn--ygbi2ammx xn--zfr164b xxx xyz yachts yandex ye yoga yokohama youtube yt za zip zm zone zw
  /) {
  $VALID_TLDS{$_} = 1;
}

# $VALID_TLDS_RE
# %VALID_TLDS as Regexp::List optimized regexp, for use in Plugins etc
# bash line to generate regex from TLD list
# Fetches domains, drops the top commet line, builds a regex from the list of domains, then formats it to remove (?-xsim:) regex modifier flags
#   wget http://data.iana.org/TLD/tlds-alpha-by-domain.txt -O - | tail -n+2 | perl -MRegexp::List -e '$/=undef; $_=<>; $r = Regexp::List->new; push @l, $_ for (split); print $r->list2re(@l)' | perl -pe 's/^\(\?[^:]*:(.*)\)$/$1/' && echo
#
$VALID_TLDS_RE = qr/
(?:X(?:N--(?:MGB(?:A(?:(?:3A4F16|YH7GP)A|AM7A8H|B2BD)|ERP4A5D4AR|C0A9AZCG|BH1A71E|X4CD0AB|9AWBF)|F(?:IQ(?:(?:228C5H|S8|Z9)S|64B)|PCRJ9C3D|ZC2C9E2C|LW351E)|C(?:ZR(?:694B|S0T|U2D)|LCHC0EA0B2G2A9GCD|G4BKI|1AVG)|V(?:ERMGENSBERAT(?:UNG-PW|ER-CT)B|HQUV)|X(?:KC2(?:DL3A5EE0H|AL3HYE2A)|HQ521B)|(?:Q(?:CKA1PM|9JYB4)|GECRJ9|90A3A)C|N(?:QV7F(?:S00EMA)?|GBC5AZD|ODE)|80A(?:S(?:EHDB|WG)|DXHKS|O21A)|3(?:E0B707E|BST00M|DS443G)|4(?:5(?:BRJ9|Q11)C|GBRIM)|KP(?:R(?:W13|Y57)D|UT3I)|P(?:1A(?:CF|I)|GBS0DH)|Y(?:FRO4I67O|GBI2AMMX)|6(?:QQ986B3XL|FRZ82G)|I(?:1B6B1A6A2E|O0A7I)|L(?:GBBAT1AD8J|1ACC)|H(?:2BRJ9C|XT814E)|O(?:GBPF8FL|3CW4H)|S(?:9BRJ9C|ES554G)|J(?:6W193G|1AMH)|55Q(?:W42G|X5D)|D1A(?:CJ3B|LF)|WGB(?:H1C|L6A)|1QQW23A|RHQV96G|ZFR164B|UNUP4Y)|XX|YZ)|C(?:[CDFGKMNVWXZ]|O(?:N(?:S(?:TRUCTION|ULTING)|(?:TRACTOR|DO)S)|M(?:P(?:UTER|ANY)|MUNITY)?|(?:L(?:LEG|OGN)|FFE)E|O(?:[LP]|KING)|UNTRY|ACH|DES)?|A(?:[BL]|R(?:E(?:ERS?)?|AVAN|TIER|DS)|P(?:ETOWN|ITAL)|NCERRESEARCH|T(?:ERING)?|M(?:ERA|P)|S[AH])?|L(?:(?:EAN|OTH)ING|I(?:NIC|CK)|AIMS|UB)?|H(?:R(?:ISTMAS|OME)|ANNEL|URCH|EAP)?|R(?:EDIT(?:CARD)?|(?:UISE)?S|ICKET)?|I(?:T(?:IC|Y))?|E(?:NTER|RN|O)|U(?:ISINELLA)?|Y(?:MRU)?)|S(?:[BDGJLMNRTVXZ]|U(?:PP(?:L(?:IES|Y)|ORT)|R(?:GERY|F)|ZUKI)?|C(?:[AB]|H(?:MIDT|WARZ|ULE)|IENCE|OT)?|O(?:L(?:UTIONS|AR)|FTWARE|CIAL|HU|Y)?|A(?:ARLAND|MSUNG|RL)?|E(?:RVICES|XY|W)?|Y(?:STEMS|DNEY)?|H(?:IKSHA|OES)?|P(?:IEGEL|ACE)|I(?:NGLES)?|KY?)|A(?:[FMWZ]|C(?:T(?:IVE|OR)|COUNTANTS|ADEMY)?|U(?:CTION|DIO|TOS)?|L(?:LFINANZ|SACE)?|S(?:SOCIATES|IA)?|R(?:CHI|MY|PA)?|Q(?:UARELLE)?|I(?:RFORCE)?|T(?:TORNEY)?|(?:BOGAD)?O|N(?:DROID)?|G(?:ENCY)?|D(?:ULT)?|E(?:RO)?|XA?)|P(?:[EFGKMNSTWY]|R(?:O(?:D(?:UCTIONS)?|PERT(?:IES|Y)|F)?|AXI|ESS)?|H(?:OTO(?:GRAPHY|S)?|ARMACY|YSIO)?|A(?:R(?:T(?:(?:NER)?S|Y)|IS))?|I(?:C(?:TURE)?S|ZZA|NK)|O(?:KER|HL|RN|ST)|L(?:UMBING|ACE)?|UB)|M(?:[CDGHKLMNPQRSTVWXYZ]|O(?:[EV]|R(?:TGAGE|MON)|N(?:ASH|EY)|TORCYCLES|SCOW|BI|DA)?|A(?:N(?:AGEMENT|GO)|RKET(?:ING)?|DRID|ISON)?|E(?:M(?:ORIAL|E)|LBOURNE|DIA|ET|NU)?|I(?:(?:AM|N)I|L)|U(?:SEUM)?)|B(?:[BDFGHJSTVWY]|U(?:ILD(?:ERS)?|DAPEST|SINESS|ZZ)|L(?:ACK(?:FRIDAY)?|OOMBERG|UE)|A(?:R(?:GAINS)?|YERN|ND)?|E(?:RLIN|ER|ST)?|I(?:[DOZ]|KE)?|N(?:PPARIBAS)?|O(?:UTIQUE|O)?|R(?:USSELS)?|MW?|ZH?)|G(?:[DFGHNPQSTWY]|R(?:A(?:PHIC|TI)S|EEN|IPE)?|L(?:OB(?:AL|O)|ASS|E)?|A(?:L(?:LERY)?|RDEN)?|U(?:I(?:TARS|DE)|RU)?|I(?:FTS?|VES)?|M(?:[OX]|AIL)?|O(?:[PV]|OGLE)|B(?:IZ)?|E(?:NT)?)|F(?:[JM]|I(?:NANC(?:IAL|E)|SH(?:ING)?|RMDALE|TNESS)?|L(?:SMIDTH|IGHTS|ORIST|Y)|O(?:UNDATION|RSALE|O)?|U(?:RNITURE|TBOL|ND)|A(?:SHION|IL|RM)|R(?:OGANS|L)?|(?:EEDBAC)?K)|E(?:[CEGR]|N(?:GINEER(?:ING)?|TERPRISES|ERGY)|X(?:P(?:OSED|ERT)|CHANGE)|U(?:ROVISION|S)?|(?:QUIPMEN|A)?T|VE(?:RBANK|NTS)|DU(?:CATION)?|M(?:ERCK|AIL)|S(?:TATE|Q)?)|D(?:[JKMZ]|E(?:NT(?:IST|AL)|LIVERY|MOCRAT|GREE|ALS|SI)?|I(?:(?:SCOUN|E)T|RECT(?:ORY)?|AMONDS|GITAL)|A(?:[DY]|TING|NCE)|O(?:MAINS|OSAN)?|URBAN|VAG|NP)|R(?:E(?:P(?:UBLICAN|AIR|ORT)|(?:CIPE|VIEW)S|S(?:TAURAN)?T|I(?:SEN?|T)|N(?:TALS)?|ALTOR|HAB|D)?|O(?:CKS|DEO)?|I(?:[OP]|CH)|S(?:VP)?|U(?:HR)?|YUKYU|W)|T(?:[CDFGHJKLMNPTVWZ]|O(?:(?:OL|Y)S|DAY|KYO|WN|P)?|R(?:A(?:INING|VEL|DE)|UST)?|A(?:T(?:TOO|AR)|IPEI|X)|I(?:ENDA|ROL|PS)|E(?:CHNOLOGY|L)|UI)|L(?:[BCKRVY]|I(?:M(?:ITED|O)|GHTING|DL|FE|NK)?|A(?:CAIXA|TROBE|WYER|ND)?|O(?:NDON|ANS|TTO)|U(?:X(?:URY|E))?|E(?:ASE|GAL)|T(?:DA)?|D?S|GBT)|V(?:[CGU]|E(?:(?:NTURE|GA)S|RSICHERUNG|T)?|O(?:T(?:[EO]|ING)|YAGE|DKA)|I(?:(?:AJE|LLA)S|SION)?|(?:LAANDERE)?N|A(?:CATIONS)?)|I(?:[DELOQST]|N(?:[GK]|(?:VESTMENT|DUSTRIE)S|T(?:ERNATIONAL)?|S(?:TITUT|UR)E|FO)?|M(?:MO(?:BILIEN)?)?|R(?:ISH)?|BM|WC)|H(?:[KMNRTU]|O(?:L(?:DINGS|IDAY)|ST(?:ING)?|[RU]SE|MES|W)|E(?:(?:ALTHCA)?RE|LP)|A(?:MBURG|US)|I(?:PHOP|V))|W(?:[FS]|E(?:B(?:SITE|CAM)|D(?:DING)?)|I(?:LLIAMHILL|EN|KI)|A(?:LES|TCH|NG)|OR(?:KS?|LD)|HOSWHO|T[CF]|ME)|N(?:[FLOPUZ]|E(?:T(?:WORK)?|USTAR|XUS|W)?|A(?:GOYA|ME|VY)?|I(?:NJA)?|R[AW]?|GO?|Y?C|HK)|K(?:[EGHMPWYZ]|I(?:TCHEN|WI|M)?|(?:AUFE|OEL)?N|R(?:E?D)?)|O(?:(?:(?:TSU|SA)K|KINAW)A|RG(?:ANIC)?|N[GL]|OO|VH|M)|Y(?:[ET]|O(?:(?:KOHAM|G)A|UTUBE)|A(?:CHTS|NDEX))|J(?:[MP]|O(?:B(?:URG|S))?|E(?:TZT)?|UEGOS)|U(?:[AGKSYZ]|N(?:IVERSITY|O)|OL)|Z(?:[AMW]|ONE|IP)|Q(?:UEBEC|PON|A))/ix;

# Two-Level TLDs
#
# to resort this, pump the whole list through:
#  perl -e '$/=undef; $_=<>; foreach(split) { ($a,$b) = split(/\./, $_, 2); $t{$b}->{$_}=1; } foreach (sort keys %t) { print "  ",join(" ", sort keys %{$t{$_}}),"\n" }'
#
# http://www.neustar.us/policies/docs/rfc_1480.txt
# data originally from http://spamcheck.freeapp.net/two-level-tlds
# The freeapp.net site now says that information on the site is obsolete
# See discussion and sources in comments of bug 5677
# updated as per bug 5815
# cleanup in progress per bug 6795 (axb)
# Unsorted sources:
# .ua : http://hostmaster.ua
# .hu : http://www.domain.hu/domain/English/szabalyzat/sld.html
#
foreach(qw/
{'fraction_non_alphanumeric': 0.19454987419506162, 'fraction_numerical': 0.03049170540321549, 'mean_word_length': 6.7011494252873565, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '128596', 'n_tokens_mistral': 10507, 'n_tokens_neox': 9963, 'n_words': 2283}