<commit_before>import json, re
import HTMLParser

import requests

from CommandTemplate import CommandTemplate
import Constants
import GlobalStore
from util import DateTimeUtil
from IrcMessage import IrcMessage

class Command(CommandTemplate):

	triggers = ['http://', 'https://', 'www']
	helptext = "Shows the title of the page somebody just posted a link to"
	showInCommandList = False
	callInThread = True  #We can't know how slow sites are, so prevent the bot from locking up on slow sites

	def shouldExecute(self, message):
		if message.messageType != 'say':
			return False
		for trigger in self.triggers:
			if trigger in message.message:
				return True
		return False

	def execute(self, message):
		"""
		:type message: IrcMessage
		"""
		timeout = 5.0  #Time to try and get a title, in seconds
		urlmatch = re.search(r"(https?://\S+)", message.message)
		if not urlmatch:
			self.logWarning("[url] Module triggered, but no url found in message '{}'".format(message.rawText))
		else:
			url = urlmatch.group()
			while url.endswith(")") or url.endswith('/'):
				url = url[:-1]

			title = None
			try:
				#There's some special cases for often used pages.
				if 'twitch.tv' in url:
					title = self.retrieveTwitchTitle(url, timeout)
				elif 'youtube.com' in url or 'youtu.be' in url:
					title = self.retrieveYoutubetitle(url, timeout)
				elif 'imgur.com' in url:
					title = self.retrieveImgurTitle(url, timeout)
				elif 'twitter.com' in url:
					title = self.retrieveTwitterTitle(url, timeout)
				elif re.match('https?://.{2}(?:\.m)?\.wikipedia.org', url, re.IGNORECASE):
					title = GlobalStore.commandhandler.runCommandFunction('getWikipediaArticle', None, url, False)
				#If nothing has been found so far, just display whatever is between the <title> tags
				if title is None:
					title = self.retrieveGenericTitle(url, timeout)
			except requests.exceptions.Timeout:
				self.logError("[url] '{}' took too long to respond, ignoring".format(url))
			except requests.exceptions.ConnectionError as error:
				self.logError("[url] A connection error occurred while trying to retrieve '{}': {}".format(url, error))

			#Finally, display the result of all the hard work, if there was any
			if title is not None:
				title = title.replace('\n', Constants.GREY_SEPARATOR).strip()
				#Convert weird characters like &#39 back into normal ones like '
				title = HTMLParser.HTMLParser().unescape(title)
				#Make sure titles aren't too long
				if len(title) > 250:
					title = title[:250] + "[...]"
				message.reply(u"Title: {}".format(title), "say")

	@staticmethod
	def retrieveGenericTitle(url, timeout=5.0):
		for ext in ('.jpg', '.jpeg', '.gif', '.png', '.bmp', '.avi', '.wav', '.mp3', '.ogg', '.zip', '.rar', '.7z', '.pdf', '.swf'):
			if url.endswith(ext):
				return None
		titlematch = re.search(r'<title ?.*?>(.+)</title>', requests.get(url, timeout=timeout).text, re.DOTALL | re.IGNORECASE)
		if titlematch:
			return titlematch.group(1)  #No need to do clean-up, that's handled in the main 'execute' function
		return None

	@staticmethod
	def retrieveTwitchTitle(url, timeout=5.0):
		channelmatches = re.search("https?://w*\.twitch\.tv/([^/]+)", url)
		if channelmatches:
			channel = channelmatches.group(1)
			channeldata = {}
			isChannelOnline = False
			twitchheaders = {'Accept': 'application/vnd.twitchtv.v2+json'}
			twitchStreamPage = requests.get(u"https://api.twitch.tv/kraken/streams/" + channel, headers=twitchheaders, timeout=timeout)
			streamdata = json.loads(twitchStreamPage.text.encode('utf-8'))
			if 'stream' in streamdata and streamdata['stream'] is not None:
				channeldata = streamdata['stream']['channel']
				isChannelOnline = True
			elif 'error' not in streamdata:<commit_msg>[urlTitleFinder] Move title cleanup to separate method<commit_after>import json, re
import HTMLParser

import requests

from CommandTemplate import CommandTemplate
import Constants
import GlobalStore
from util import DateTimeUtil
from IrcMessage import IrcMessage

class Command(CommandTemplate):

	triggers = ['http://', 'https://', 'www']
	helptext = "Shows the title of the page somebody just posted a link to"
	showInCommandList = False
	callInThread = True  #We can't know how slow sites are, so prevent the bot from locking up on slow sites

	def shouldExecute(self, message):
		if message.messageType != 'say':
			return False
		for trigger in self.triggers:
			if trigger in message.message:
				return True
		return False

	def execute(self, message):
		"""
		:type message: IrcMessage
		"""
		timeout = 5.0  #Time to try and get a title, in seconds
		urlmatch = re.search(r"(https?://\S+)", message.message)
		if not urlmatch:
			self.logWarning("[url] Module triggered, but no url found in message '{}'".format(message.rawText))
		else:
			url = urlmatch.group()
			while url.endswith(")") or url.endswith('/'):
				url = url[:-1]

			title = None
			try:
				#There's some special cases for often used pages.
				if 'twitch.tv' in url:
					title = self.retrieveTwitchTitle(url, timeout)
				elif 'youtube.com' in url or 'youtu.be' in url:
					title = self.retrieveYoutubetitle(url, timeout)
				elif 'imgur.com' in url:
					title = self.retrieveImgurTitle(url, timeout)
				elif 'twitter.com' in url:
					title = self.retrieveTwitterTitle(url, timeout)
				elif re.match('https?://.{2}(?:\.m)?\.wikipedia.org', url, re.IGNORECASE):
					title = GlobalStore.commandhandler.runCommandFunction('getWikipediaArticle', None, url, False)
				#If nothing has been found so far, just display whatever is between the <title> tags
				if title is None:
					title = self.retrieveGenericTitle(url, timeout)
			except requests.exceptions.Timeout:
				self.logError("[url] '{}' took too long to respond, ignoring".format(url))
			except requests.exceptions.ConnectionError as error:
				self.logError("[url] A connection error occurred while trying to retrieve '{}': {}".format(url, error))

			#Finally, display the result of all the hard work, if there was any
			if title is not None:
				title = Command.cleanUpRetrievedTitle(title)
				message.reply(u"Title: {}".format(title))

	@staticmethod
	def cleanUpRetrievedTitle(retrievedTitle):
		cleanedUpTitle = retrievedTitle.strip()
		cleanedUpTitle = cleanedUpTitle.replace(' *\n *', Constants.GREY_SEPARATOR)
		# Convert weird characters like &#39 back into normal ones like '
		cleanedUpTitle = HTMLParser.HTMLParser().unescape(cleanedUpTitle)
		# Make sure titles aren't too long
		if len(cleanedUpTitle) >= Constants.MAX_MESSAGE_LENGTH:
			cleanedUpTitle = cleanedUpTitle[:Constants.MAX_MESSAGE_LENGTH - 5] + "[...]"
		return cleanedUpTitle

	@staticmethod
	def retrieveGenericTitle(url, timeout=5.0):
		for ext in ('.jpg', '.jpeg', '.gif', '.png', '.bmp', '.avi', '.wav', '.mp3', '.ogg', '.zip', '.rar', '.7z', '.pdf', '.swf'):
			if url.endswith(ext):
				return None
		titlematch = re.search(r'<title ?.*?>(.+)</title>', requests.get(url, timeout=timeout).text, re.DOTALL | re.IGNORECASE)
		if titlematch:
			return titlematch.group(1)  #No need to do clean-up, that's handled in the main 'execute' function
		return None

	@staticmethod
	def retrieveTwitchTitle(url, timeout=5.0):
		channelmatches = re.search("https?://w*\.twitch\.tv/([^/]+)", url)
		if channelmatches:
			channel = channelmatches.group(1)
			channeldata = {}
			isChannelOnline = False
			twitchheaders = {'Accept': 'application/vnd.twitchtv.v2+json'}
			twitchStreamPage = requests.get(u"https://api.twitch.tv/kraken/streams/" + channel, headers=twitchheaders, timeout=timeout)
			streamdata = json.loads(twitchStreamPage.text.encode('utf-8'))
			if 'stream' in streamdata and streamdata['stream'] is not None:
				channeldata = streamdata['stream']['channel']
				isChannelOnline = True
			elif 'error' not in streamdata:
{'fraction_non_alphanumeric': 0.1343762020771894, 'fraction_numerical': 0.005000641107834338, 'mean_word_length': 4.091383812010444, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2102312', 'n_tokens_mistral': 2765, 'n_tokens_neox': 2423, 'n_words': 739}