<commit_before><?xml version="1.0" encoding="UTF-8"?>
<data xmlns="http://www.woltlab.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.woltlab.com http://www.woltlab.com/XSD/5.4/spiderList.xsd">
	<spider ident="ABCdatos">
		<name>ABCdatos BotLink</name>
		<url>http://www.robotstxt.org/db/abcdatos.html</url>
	</spider>
	<spider ident="AbraveSpider">
		<name>Abrave</name>
		<url>http://robot.abrave.co.uk/</url>
	</spider>
	<spider ident="abot/">
		<name>abot</name>
	</spider>
	<spider ident="Accelatech RSSCrawler">
		<name>Accelatech</name>
	</spider>
	<spider ident="Accoona-AI-Agent">
		<name>Accoona</name>
		<url>https://www.accoona.com/about/</url>
	</spider>
	<spider ident="aconon Index">
		<name>aconon Index (raubfische.de)</name>
	</spider>
	<spider ident="AcoonBot">
		<name>Acoon</name>
		<url>http://www.acoon.de/robot.asp</url>
	</spider>
	<spider ident="Ahoy!">
		<name>Ahoy!</name>
		<url>http://www.robotstxt.org/db/ahoythehomepagefinder.html</url>
	</spider>
	<spider ident="AhrefsBot">
		<name>Ahrefs</name>
		<url>http://ahrefs.com/robot/</url>
	</spider>
	<spider ident="AlkalineBOT">
		<name>Alkaline</name>
		<url>http://www.robotstxt.org/db/Alkaline.html</url>
	</spider>
	<spider ident="Amazonbot">
		<name>Amazonbot</name>
		<url>https://developer.amazon.com/support/amazonbot</url>
	</spider>
	<spider ident="AlexaBOT">
		<name>Alexa</name>
		<url>http://www.alexa.com/help/webmasters</url>
	</spider>
	<spider ident="http://www.almaden.ibm.com/cs/crawler">
		<name>Almaden Crawler</name>
		<url>http://www.almaden.ibm.com/cs/crawler/</url>
	</spider>
	<spider ident="Barkrowler">
		<name>Barkrowler</name>
		<url>https://www.babbar.tech/crawler</url>
	</spider>
	<spider ident="EMC Spider">
		<name>ananzi</name>
	</spider>
	<spider ident="Anthill">
		<name>Anthill</name>
		<url>http://www.robotstxt.org/db/anthill.html</url>
	</spider>
	<spider ident="Aport">
		<name>Aport</name>
		<url>http://www.aport.ru/</url>
	</spider>
	<spider ident="AppleSyndication">
		<name>Apple</name>
	</spider>
	<spider ident="Applebot">
		<name>Applebot</name>
		<url>https://support.apple.com/en-gb/HT204683</url>
	</spider>
	<spider ident="Arachnophilia">
		<name>Arachnophilia</name>
		<url>http://www.robotstxt.org/db/arachnophilia.html</url>
	</spider>
	<spider ident="Araneo">
		<name>Araneo</name>
		<url>http://www.robotstxt.org/db/araneo.html</url>
	</spider>
	<spider ident="ArchitextSpider">
		<name>ArchitextSpider</name>
		<url>http://www.robotstxt.org/db/architext.html</url>
	</spider>
	<spider ident="archive.org_bot">
		<name>Archive.org</name>
		<url>https://archive.org/details/archive.org_bot</url>
	</spider>
	<spider ident="arks/1.0">
		<name>arks</name>
		<url>http://www.robotstxt.org/db/arks.html</url>
	</spider>
	<spider ident="ASpider">
		<name>ASpider</name>
		<url>http://www.robotstxt.org/db/aspider.html</url>
	</spider>
	<spider ident="ATN_Worldwide">
		<name>ATN Worldwide</name>
		<url>http://www.robotstxt.org/db/atn.html</url>
	</spider>
	<spider ident="Atomz">
		<name>Atomz.com</name>
		<url>http://www.robotstxt.org/db/atomz.html</url>
	</spider>
	<spider ident="AURESYS">
		<name>AURESYS</name>
		<url>http://www.robotstxt.org/db/auresys.html</url>
	</spider>
	<spider ident="BackRub">
		<name>BackRub</name>
		<url>http://www.robotstxt.org/db/backrub.html</url>
	</spider>
	<spider ident="Baiduspider">
		<name>Baiduspider</name>
		<url>http://www.baidu.com/search/spider.htm</url>
	</spider>
	<spider ident="BecomeBot">
		<name>BecomeBot</name>
		<url>http://www.become.com/webmasters.html</url>
	</spider>
	<spider ident="Big Brother">
		<name>Big Brother</name><commit_msg>Add `AwarioBot` to spiderList.xml<commit_after><?xml version="1.0" encoding="UTF-8"?>
<data xmlns="http://www.woltlab.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.woltlab.com http://www.woltlab.com/XSD/5.4/spiderList.xsd">
	<spider ident="ABCdatos">
		<name>ABCdatos BotLink</name>
		<url>http://www.robotstxt.org/db/abcdatos.html</url>
	</spider>
	<spider ident="AbraveSpider">
		<name>Abrave</name>
		<url>http://robot.abrave.co.uk/</url>
	</spider>
	<spider ident="abot/">
		<name>abot</name>
	</spider>
	<spider ident="Accelatech RSSCrawler">
		<name>Accelatech</name>
	</spider>
	<spider ident="Accoona-AI-Agent">
		<name>Accoona</name>
		<url>https://www.accoona.com/about/</url>
	</spider>
	<spider ident="aconon Index">
		<name>aconon Index (raubfische.de)</name>
	</spider>
	<spider ident="AcoonBot">
		<name>Acoon</name>
		<url>http://www.acoon.de/robot.asp</url>
	</spider>
	<spider ident="Ahoy!">
		<name>Ahoy!</name>
		<url>http://www.robotstxt.org/db/ahoythehomepagefinder.html</url>
	</spider>
	<spider ident="AhrefsBot">
		<name>Ahrefs</name>
		<url>http://ahrefs.com/robot/</url>
	</spider>
	<spider ident="AlkalineBOT">
		<name>Alkaline</name>
		<url>http://www.robotstxt.org/db/Alkaline.html</url>
	</spider>
	<spider ident="Amazonbot">
		<name>Amazonbot</name>
		<url>https://developer.amazon.com/support/amazonbot</url>
	</spider>
	<spider ident="AlexaBOT">
		<name>Alexa</name>
		<url>http://www.alexa.com/help/webmasters</url>
	</spider>
	<spider ident="http://www.almaden.ibm.com/cs/crawler">
		<name>Almaden Crawler</name>
		<url>http://www.almaden.ibm.com/cs/crawler/</url>
	</spider>
	<spider ident="Barkrowler">
		<name>Barkrowler</name>
		<url>https://www.babbar.tech/crawler</url>
	</spider>
	<spider ident="EMC Spider">
		<name>ananzi</name>
	</spider>
	<spider ident="Anthill">
		<name>Anthill</name>
		<url>http://www.robotstxt.org/db/anthill.html</url>
	</spider>
	<spider ident="Aport">
		<name>Aport</name>
		<url>http://www.aport.ru/</url>
	</spider>
	<spider ident="AppleSyndication">
		<name>Apple</name>
	</spider>
	<spider ident="Applebot">
		<name>Applebot</name>
		<url>https://support.apple.com/en-gb/HT204683</url>
	</spider>
	<spider ident="Arachnophilia">
		<name>Arachnophilia</name>
		<url>http://www.robotstxt.org/db/arachnophilia.html</url>
	</spider>
	<spider ident="Araneo">
		<name>Araneo</name>
		<url>http://www.robotstxt.org/db/araneo.html</url>
	</spider>
	<spider ident="ArchitextSpider">
		<name>ArchitextSpider</name>
		<url>http://www.robotstxt.org/db/architext.html</url>
	</spider>
	<spider ident="archive.org_bot">
		<name>Archive.org</name>
		<url>https://archive.org/details/archive.org_bot</url>
	</spider>
	<spider ident="arks/1.0">
		<name>arks</name>
		<url>http://www.robotstxt.org/db/arks.html</url>
	</spider>
	<spider ident="ASpider">
		<name>ASpider</name>
		<url>http://www.robotstxt.org/db/aspider.html</url>
	</spider>
	<spider ident="ATN_Worldwide">
		<name>ATN Worldwide</name>
		<url>http://www.robotstxt.org/db/atn.html</url>
	</spider>
	<spider ident="Atomz">
		<name>Atomz.com</name>
		<url>http://www.robotstxt.org/db/atomz.html</url>
	</spider>
	<spider ident="AURESYS">
		<name>AURESYS</name>
		<url>http://www.robotstxt.org/db/auresys.html</url>
	</spider>
	<spider ident="AwarioSmartBot">
		<name>AwarioSmartBot</name>
		<url>https://awario.com/bots.html</url>
	</spider>
	<spider ident="AwarioRssBot">
		<name>AwarioRssBot</name>
		<url>https://awario.com/bots.html</url>
	</spider>
	<spider ident="BackRub">
		<name>BackRub</name>
		<url>http://www.robotstxt.org/db/backrub.html</url>
	</spider>
	<spider ident="Baiduspider">
		<name>Baiduspider</name>
		<url>http://www.baidu.com/search/spider.htm</url>
	</spider>
	<spider ident="BecomeBot">
		<name>BecomeBot</name>
		<url>http://www.become.com/webmasters.html</url>
	</spider>
	<spider ident="Big Brother">
		<name>Big Brother</name>
{'fraction_non_alphanumeric': 0.22144736842105264, 'fraction_numerical': 0.004736842105263158, 'mean_word_length': 4.6597170513775135, 'pattern_counts': {'":': 0, '<': 377, '<?xml version=': 2, '>': 377, 'https://': 12, 'lorem ipsum': 0, 'www.': 52, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '782434', 'n_tokens_mistral': 3594, 'n_tokens_neox': 3383, 'n_words': 354}