<commit_before>import discord, random, time
import datetime as dt
from discord.ext import commands
from urllib.parse import unquote
from html.parser import HTMLParser
from Cogs import DL, Message

def setup(bot):
	settings = bot.get_cog("Settings")
	bot.add_cog(Comic(bot, settings))

class Comic(commands.Cog):

	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings
		self.max_tries = 10
		self.comic_data = {
			"calvin": {
				"name": "<NAME>",
				"url": "https://www.gocomics.com/calvinandhobbes/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "11-18-1985",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"cyanide": {
				"name": "Cyanide & Happiness",
				"url": "https://explosm.net/comics/{}/",
				"comic_number": True,
				"first_date": 15,
				"archive_url": "http://explosm.net/comics/archive/{}/{}",
				"archive_keys": ["year","month"],
				"latest_url": [
					{"find":'id="comic-author">',"index":0},
					{"find":'a href="/comics/',"index":-1},
					{"find":'/"',"index":0}
				],
				"date_url": [
					{"find":"{}.{}.{}","keys":["year","month","day"],"index":0},
					{"find":'a href="/comics/',"index":-1},
					{"find":'/"',"index":0}
				],
				"comic_url": [
					{"find":'"main-comic" src="',"index":-1},
					{"find":'"',"index":0}
				],
			},
			"dilbert": {
				"name": "Dilbert",
				"url": "https://dilbert.com/strip/{}-{}-{}",
				"keys": ["year","month","day"],
				"first_date": "04-16-1989",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				],
				"comic_title": [
					{"find":'data-title="',"index":-1},
					{"find":'"',"index":0},
					{"find":"Dilbert Comic for ","index":0}
				]
			},
			"garfield": {
				"name": "Garfield",
				"url": "https://www.gocomics.com/garfield/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "06-19-1978",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"gmg": {
				"name": "Garfield Minus Garfield",
				"url": "https://garfieldminusgarfield.net/day/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "02-13-2008",
				"comic_url": [
					{"find":'<img src="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"peanuts": {
				"name": "Peanuts",
				"url": "https://www.gocomics.com/peanuts/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "10-02-1950",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"xkcd": {
				"name": "XKCD",
				"comic_number": True,
				"first_date": 1,
				"archive_url": "https://xkcd.com/archive/",
				"latest_url": [
					{"find":"<h1>Comics:</h1>","index":-1},
					{"find":'a href="/',"index":1},
					{"find":'/" title="',"index":0}
				],
				"date_url": [
					{"find":'title="{}-{}-{}"',"keys":["year","month","day"],"index":0},
					{"find":'a href="/',"index":-1},
					{"find":'/"',"index":0}
				],
				"url": "https://xkcd.com/{}/",
				"comic_url": [
					{"find":"(for hotlinking/embedding): ","index":1},
					{"find":"\n","index":0},
					{"find":'<a href= "',"index":-1},
					{"find":'">',"index":0}
				],
				"comic_desc": [
					{"find":'<div id="comic">',"index":-1},
					{"find":'title="',"index":1},
					{"find":'"',"index":0}
				],
				"comic_title": [
					{"find":'<div id="comic">',"index":-1},
					{"find":'alt="',"index":1},
					{"find":'"',"index":0}
				],
				"padded": False
			}
		}

	def _julian_day(self,gregorian_day):
		# Takes a date string MM-DD-YYYY and returns the Julian day
		M,D,Y = [int(x) for x in gregorian_day.split("-")]
		return int(dt.date(Y,M,D).toordinal() + 1721424.5)

	def _gregorian_day(self,julian_day):
		# Takes a Julian day and returns MM-DD-YYYY in Gregorian
		return dt.date.fromordinal(int(julian_day-1721424.5)).strftime("%m-%d-%Y")

	def _date_dict(self,date,padded=True):
		if isinstance(date,(int,float)):
			date = self._gregorian_day(date)
		m,d,y = [str(int(x)).rjust(2,"0") if padded else str(x) for x in date.split("-")]
		return {"month":m,"day":d,"year":y}

	async def _get_last_comic_number(self,comic_data,date=None):
		# Helper to return the highest comic number for a given comic and source html
		date_dict = self._date_dict(dt.datetime.today().strftime("%m-%d-%Y") if date == None else date,padded=comic_data.get("padded",True))
		try: 
			archive_url = comic_data["archive_url"].format(*[date_dict[x] for x in comic_data.get("archive_keys",[])])
			archive_html = await DL.async_text(archive_url)
		except:
			return (None,None)
		latest_comic = self._walk_replace(archive_html,comic_data["latest_url"])
		if not latest_comic: return (None,None)
		return (int(latest_comic),archive_html)

	async def _get_random_comic(self,comic_data):
		# Try to get a random comic between the first_date/last_date, or between custom indexes (XKCD)
		latest_tuple = None
		use_number = comic_data.get("comic_number",False)
		if use_number:
			# We're using numbers - not dates
			latest_tuple = await self._get_last_comic_number(comic_data)
			if latest_tuple[0] == None: return None # borken
			first = comic_data["first_date"]
			last  = latest_tuple[0]
		else:
			# Using dates, organize them into julian days
			first = self._julian_day(comic_data["first_date"])
			last  = self._julian_day(comic_data.get("last_date",dt.datetime.today().strftime("%m-%d-%Y")))
		for x in range(self.max_tries):
			# Generate a random date
			date = random.randint(first,last)
			if not use_number: date = self._gregorian_day(date)
			comic = await self._get_comic(comic_data,date,latest_tuple)
			if comic: return comic
		return None

	def _walk_replace(self,search_text,steps,key_dict=None):
		text = search_text
		for step in steps:
			try:<commit_msg>Roll back up to 10 months when getting latest<commit_after>import discord, random, time
import datetime as dt
from discord.ext import commands
from urllib.parse import unquote
from html.parser import HTMLParser
from Cogs import DL, Message

def setup(bot):
	settings = bot.get_cog("Settings")
	bot.add_cog(Comic(bot, settings))

class Comic(commands.Cog):

	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings
		self.max_tries = 10
		self.comic_data = {
			"calvin": {
				"name": "<NAME>",
				"url": "https://www.gocomics.com/calvinandhobbes/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "11-18-1985",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"cyanide": {
				"name": "Cyanide & Happiness",
				"url": "https://explosm.net/comics/{}/",
				"comic_number": True,
				"first_date": 15,
				"archive_url": "http://explosm.net/comics/archive/{}/{}",
				"archive_keys": ["year","month"],
				"latest_url": [
					{"find":'id="comic-author">',"index":0},
					{"find":'a href="/comics/',"index":-1},
					{"find":'/"',"index":0}
				],
				"date_url": [
					{"find":"{}.{}.{}","keys":["year","month","day"],"index":0},
					{"find":'a href="/comics/',"index":-1},
					{"find":'/"',"index":0}
				],
				"comic_url": [
					{"find":'"main-comic" src="',"index":-1},
					{"find":'"',"index":0}
				],
			},
			"dilbert": {
				"name": "Dilbert",
				"url": "https://dilbert.com/strip/{}-{}-{}",
				"keys": ["year","month","day"],
				"first_date": "04-16-1989",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				],
				"comic_title": [
					{"find":'data-title="',"index":-1},
					{"find":'"',"index":0},
					{"find":"Dilbert Comic for ","index":0}
				]
			},
			"garfield": {
				"name": "Garfield",
				"url": "https://www.gocomics.com/garfield/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "06-19-1978",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"gmg": {
				"name": "Garfield Minus Garfield",
				"url": "https://garfieldminusgarfield.net/day/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "02-13-2008",
				"comic_url": [
					{"find":'<img src="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"peanuts": {
				"name": "Peanuts",
				"url": "https://www.gocomics.com/peanuts/{}/{}/{}",
				"keys": ["year","month","day"],
				"first_date": "10-02-1950",
				"comic_url": [
					{"find":'data-image="',"index":-1},
					{"find":'"',"index":0}
				]
			},
			"xkcd": {
				"name": "XKCD",
				"comic_number": True,
				"first_date": 1,
				"archive_url": "https://xkcd.com/archive/",
				"latest_url": [
					{"find":"<h1>Comics:</h1>","index":-1},
					{"find":'a href="/',"index":1},
					{"find":'/" title="',"index":0}
				],
				"date_url": [
					{"find":'title="{}-{}-{}"',"keys":["year","month","day"],"index":0},
					{"find":'a href="/',"index":-1},
					{"find":'/"',"index":0}
				],
				"url": "https://xkcd.com/{}/",
				"comic_url": [
					{"find":"(for hotlinking/embedding): ","index":1},
					{"find":"\n","index":0},
					{"find":'<a href= "',"index":-1},
					{"find":'">',"index":0}
				],
				"comic_desc": [
					{"find":'<div id="comic">',"index":-1},
					{"find":'title="',"index":1},
					{"find":'"',"index":0}
				],
				"comic_title": [
					{"find":'<div id="comic">',"index":-1},
					{"find":'alt="',"index":1},
					{"find":'"',"index":0}
				],
				"padded": False
			}
		}

	def _julian_day(self,gregorian_day):
		# Takes a date string MM-DD-YYYY and returns the Julian day
		M,D,Y = [int(x) for x in gregorian_day.split("-")]
		return int(dt.date(Y,M,D).toordinal() + 1721424.5)

	def _gregorian_day(self,julian_day):
		# Takes a Julian day and returns MM-DD-YYYY in Gregorian
		return dt.date.fromordinal(int(julian_day-1721424.5)).strftime("%m-%d-%Y")

	def _date_dict(self,date,padded=True):
		if isinstance(date,(int,float)):
			date = self._gregorian_day(date)
		m,d,y = [str(int(x)).rjust(2,"0") if padded else str(x) for x in date.split("-")]
		return {"month":m,"day":d,"year":y}

	async def _get_last_comic_number(self,comic_data,date=None,month_adjust=0):
		if month_adjust >= 10: return (None,None) # Adjusted too far :(
		today = dt.datetime.today()
		if month_adjust: # We need to adjust months
			today = dt.datetime(today.year-1,12,1) if today.month == 1 else dt.datetime(today.year,today.month-1,1)
		# Helper to return the highest comic number for a given comic and source html
		date_dict = self._date_dict(today.strftime("%m-%d-%Y") if date == None else date,padded=comic_data.get("padded",True))
		try: 
			archive_url = comic_data["archive_url"].format(*[date_dict[x] for x in comic_data.get("archive_keys",[])])
			archive_html = await DL.async_text(archive_url)
		except:
			return (None,None)
		latest_comic = self._walk_replace(archive_html,comic_data["latest_url"])
		if not latest_comic: return (None,None)
		# Try to cast the number as int - if not possible, set the month back by one
		try: latest_comic = int(latest_comic)
		except: return await self._get_last_comic_number(comic_data,date,month_adjust+1)
		return (latest_comic,archive_html)

	async def _get_random_comic(self,comic_data):
		# Try to get a random comic between the first_date/last_date, or between custom indexes (XKCD)
		latest_tuple = None
		use_number = comic_data.get("comic_number",False)
		if use_number:
			# We're using numbers - not dates
			latest_tuple = await self._get_last_comic_number(comic_data)
			if latest_tuple[0] == None: return None # borken
			first = comic_data["first_date"]
			last  = latest_tuple[0]
		else:
			# Using dates, organize them into julian days
			first = self._julian_day(comic_data["first_date"])
			last  = self._julian_day(comic_data.get("last_date",dt.datetime.today().strftime("%m-%d-%Y")))
		for x in range(self.max_tries):
			# Generate a random date
			date = random.randint(first,last)
			if not use_number: date = self._gregorian_day(date)
			comic = await self._get_comic(comic_data,date,latest_tuple)
			if comic: return comic
		return None

	def _walk_replace(self,search_text,steps,key_dict=None):
		text = search_text
		for step in steps:
			try:
{'fraction_non_alphanumeric': 0.23327007178810133, 'fraction_numerical': 0.018235828038617046, 'mean_word_length': 3.81143310837634, 'pattern_counts': {'":': 264, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 16, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6458233', 'n_tokens_mistral': 5699, 'n_tokens_neox': 4654, 'n_words': 792}