<commit_before>/*
Scrape the Wikipedia articles about Mythbusters episodes and compile a more-or-less readable form of the list.

TODO: More wiki-to-text translations, as appropriate

TODO: Should this actually parse wiki markup, instead of depending on heaps of formatting details?
*/

Stdio.File out=Stdio.File("_mythbusters.txt","wct");

void fetch(int year,string url)
{
		string data=Protocols.HTTP.get_url_data(url);
		if (!data) return;
		sscanf(data,"%*s<onlyinclude>%s</onlyinclude>",string body);
		if (!body) return; //All we care about is the stuff transcluded into the main page - not the episode details. If there is no such section, it's probably a redirect or something.
		int count=0;
		while (sscanf(body,"%*s{{Episode list\n|%s\n}}%s",string ep,body))
		{
			mapping info=([]);
			while (ep!="" && sscanf(ep,"%s=%s\n|%s",string kwd,string val,ep)==3) info[String.trim_all_whites(kwd)]=String.trim_all_whites(val);
			sscanf(info->Title||"","[[%*s|%s]]",string title);
			string myths=info->ShortSummary||""; sscanf(myths,"'''Myths tested''':%s",myths);
			while (sscanf(myths,"%s[[%s]]%s",string before,string link,string after)==3) myths=before+(link/"|")[-1]+after;
			myths-="'''Note''': This was a special episode."; //We really don't need that tag repeated everywhere...
			myths-="'''Note''': This is a special episode."; //Nor this form of it...
			out->write("/%d-%s %s.mkv\n%s\n\n",
				year,replace(info->EpisodeNumber2||"00","Special ","SP"),
				replace(title||"title unknown","/","-"),
				String.trim_all_whites(replace(myths,"<br />","\n"))
			);
			++count;
		}
		write("%d episodes.\n",count);
}

int main()
{
	fetch(2003,"http://en.wikipedia.org/w/index.php?title=List_of_MythBusters_pilot_episodes&action=raw"); //Hack: Grab the pilot episodes and count them as 2003, same as the first actual year (they count as specials)
	for (int year=2003;;++year)
	{
		write("Fetching %d... ",year);
		fetch(year,"http://en.wikipedia.org/w/index.php?title=MythBusters_("+year+"_season)&action=raw");
	}
	write("No such page found - all done.\n");
	out->close();
	Process.create_process(({"git","commit","_mythbusters.txt","-mUpdate Mythbusters from Wikipedia"}))->wait();
	Process.create_process(({"scp","_mythbusters.txt","netbios@huix:Mythbusters/00index.txt"}))->wait();
}<commit_msg>Bugfix latest changes, and backtab<commit_after>/*
Scrape the Wikipedia articles about Mythbusters episodes and compile a more-or-less readable form of the list.

TODO: More wiki-to-text translations, as appropriate

TODO: Should this actually parse wiki markup, instead of depending on heaps of formatting details?
*/

Stdio.File out=Stdio.File("_mythbusters.txt","wct");

int fetch(int year,string url)
{
	write("Fetching %d... ",year);
	string data=Protocols.HTTP.get_url_data(url);
	if (!data) return 0;
	sscanf(data,"%*s<onlyinclude>%s</onlyinclude>",string body);
	if (!body) return 0; //All we care about is the stuff transcluded into the main page - not the episode details. If there is no such section, it's probably a redirect or something.
	int count=0;
	while (sscanf(body,"%*s{{Episode list\n|%s\n}}%s",string ep,body))
	{
		mapping info=([]);
		while (ep!="" && sscanf(ep,"%s=%s\n|%s",string kwd,string val,ep)==3) info[String.trim_all_whites(kwd)]=String.trim_all_whites(val);
		sscanf(info->Title||"","[[%*s|%s]]",string title);
		string myths=info->ShortSummary||""; sscanf(myths,"'''Myths tested''':%s",myths);
		while (sscanf(myths,"%s[[%s]]%s",string before,string link,string after)==3) myths=before+(link/"|")[-1]+after;
		myths-="'''Note''': This was a special episode."; //We really don't need that tag repeated everywhere...
		myths-="'''Note''': This is a special episode."; //Nor this form of it...
		out->write("/%d-%s %s.mkv\n%s\n\n",
			year,replace(info->EpisodeNumber2||"00","Special ","SP"),
			replace(title||"title unknown","/","-"),
			String.trim_all_whites(replace(myths,"<br />","\n"))
		);
		++count;
	}
	write("%d episodes.\n",count);
	return count;
}

int main()
{
	fetch(2003,"http://en.wikipedia.org/w/index.php?title=List_of_MythBusters_pilot_episodes&action=raw"); //Hack: Grab the pilot episodes and count them as 2003, same as the first actual year (they count as specials)
	for (int year=2003;fetch(year,"http://en.wikipedia.org/w/index.php?title=MythBusters_("+year+"_season)&action=raw");++year);
	write("No such page found - all done.\n");
	out->close();
	Process.create_process(({"git","commit","_mythbusters.txt","-mUpdate Mythbusters from Wikipedia"}))->wait();
	Process.create_process(({"scp","_mythbusters.txt","netbios@huix:Mythbusters/00index.txt"}))->wait();
}
{'fraction_non_alphanumeric': 0.19728037988344485, 'fraction_numerical': 0.009497086121303691, 'mean_word_length': 5.563739376770538, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2591815', 'n_tokens_mistral': 1754, 'n_tokens_neox': 1629, 'n_words': 379}