<commit_before>#!/usr/bin/python3
import sys
import subprocess
import re
import os
import os.path
import argparse
import itertools
import logging
import urllib.request
import lxml.html
import irc.bot
import irc.client
import random
import pickle
import atexit
import configparser
import time

DEFAULTS = {
		'server': 'irc.freenode.net',
		'nickname': 'tontobot',
		'channel': '#gultec',
		'realname': 'Tonto Bot',
		'port': 6667
		}

def get_urls(s):
	return re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', s)

class TontoBot(irc.bot.SingleServerIRCBot):
	FETCH_MAX = 20 * 1024
	MSG_MAX = 140
	FAIL_MSGS = [':(', '):', '?', 'WAT', 'No pos no', 'link no worky', 'chupa limon']

	def __init__(self, serverspec, channel, nickname, realname, seen_urlpath='./seenurls.pickle'):
		irc.bot.SingleServerIRCBot.__init__(self, [serverspec], nickname, realname)
		self.channel = channel
		self.seen_urlpath = seen_urlpath
		logging.info("nickname=[%s] realname=[%s] channel=[%s]" % (nickname, realname, channel))
		try:
			with open(seen_urlpath, 'rb') as f:
				self.urlhist = pickle.load(f)
		except:
			self.urlhist = {}
		atexit.register(self._dumphist)

	def on_welcome(self, connection, event):
		logging.debug("joining %s", self.channel)
		connection.join(self.channel)

	def _sendmsg(self, connection, msg):
		"""Convenience method to send a msg. Truncates msg to MSG_MAX chars"""
		msg = msg.replace('\n', ' ')
		logging.info("msg: %s" % msg)
		if len(msg) > 140:
			msg = msg[self.MSG_MAX:]
			logging.info("truncated msg: %s" % msg)
		connection.privmsg(self.channel, msg)

	def _dumphist(self):
		try:
			with open(self.seen_urlpath, 'wb') as f:
				pickle.dump(self.urlhist, f)
		except:
			logging.exception("Oh noes, history is lost")

	def urlopen(self, url, maxbytes=FETCH_MAX):
		req = urllib.request.Request(url, headers={'User-agent': 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11'})
		fd = urllib.request.urlopen(req)
		return fd.read(maxbytes)

	def masca(self):
		openers = ('en serio que', 'neta que', 'al chile', '')
		haters = ('me caga', 'me re-caga', 'me molesta un chingo', 'está todo mal', 'me emputa', 'está todo gacho', 'es lo más ojete de la vida', 'es bien tonto')
		closers = ('que fecal fue presidente', 'que no uses C', 'que uses camelCase', 'que existas', 'que un lenguaje no tenga apuntadores', 'que amlo perdió', '')
		standalones = ('Viva el EZLN', '¬¬', '¡Viva la revolución!', 'Estás todo gacho', 'Vamos por cheve')

		composed_len = len(openers) * len(haters) * len(closers)
		totlen = composed_len + len(standalones)

		logging.info('totlen = %d' % totlen)
		logging.info('composed_len = %d' % composed_len)
		# Give equal weight to standalones and composed. This should make standalones a relatively rare occurrence
		if random.randint(0, totlen) < composed_len:
			now = time.clock()
			quote = ' '.join(random.choice(list(itertools.product(openers, haters, closers)))).strip()
			end = time.clock()
			logging.info('Took %d seconds to generate a quote' % ( now - end))
			return quote
		else:
			return random.choice(standalones)

	def rtfm(self, line):
		argv = line.split()
		if len(argv) == 3:
			cmd = argv[2].strip()
			section = argv[1]
		elif len(argv) == 2:
			cmd = argv[1].strip()
			section = None
		else:
			raise Exception("format: !rtfm [section] cmd")

		if not re.match('^[a-zA-Z_-]+$', cmd):
			logging.error("re.match rtfm: %s" % cmd)
			raise Exception("Funky commands not supported")

		if section:
			stdout = subprocess.check_output(['man', '--pager', 'cat', str(section), cmd])
		else:
			stdout = subprocess.check_output(['man', '--pager', 'cat', cmd])
		stdout = stdout.decode('utf-8')

		if stdout:
			seen_description = False
			for line in stdout.splitlines():
				line = line.strip()
				if seen_description:
					return line.split('.')[0]
				if line.endswith('DESCRIPTION'):
					seen_description = True
			raise Exception("Unable to parse manpage")

		raise Exception("No man page found")

	def on_pubmsg(self, connection, event):
		line = event.arguments[0]
		try:
			if line.startswith('!rtfm'):
				self._sendmsg(connection, self.rtfm(line))
			elif line.startswith('!masca'):
				self._sendmsg(connection, self.masca())
			elif line.startswith('ping'):
				self._sendmsg(connection, 'pong')
		except:
			logging.exception("Failed with: %s" % line)
		for u in get_urls(line):
			try:
				if u.endswith(('.jpg', '.png', '.git', '.bmp', '.pdf')):
					logging.info('not a webpage, skipping')
					continue
				root = lxml.html.fromstring(self.urlopen(u))
				title = root.find('.//title').text
				if u in self.urlhist:
					self._sendmsg(connection, '[repost] ' + title)
				else:
					self._sendmsg(connection, title)
					self.urlhist[u] = title
			except:
				logging.exception("Failed with: %s" % line)
				self._sendmsg(connection, random.choice(self.FAIL_MSGS))

def get_args():
	parser = argparse.ArgumentParser()
	parser.add_argument('--server')
	parser.add_argument('--nickname')
	parser.add_argument('--channel')
	parser.add_argument('--realname')<commit_msg>Tinify url's which are longer than MAX_URLLEN<commit_after>#!/usr/bin/python3
import sys
import subprocess
import re
import os
import os.path
import argparse
import itertools
import logging
import urllib.request
import lxml.html
import irc.bot
import irc.client
import random
import pickle
import atexit
import configparser
import time

DEFAULTS = {
		'server': 'irc.freenode.net',
		'nickname': 'tontobot',
		'channel': '#gultec',
		'realname': '<NAME>',
		'port': 6667
		}

def get_urls(s):
	return re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', s)

class TontoBot(irc.bot.SingleServerIRCBot):
	FETCH_MAX = 20 * 1024
	URL_MAXLEN = 60 # If url is longer than this, tontobot will provide a tinified version of the url
	MSG_MAX = 140
	FAIL_MSGS = [':(', '):', '?', 'WAT', 'No pos no', 'link no worky', 'chupa limon']

	def __init__(self, serverspec, channel, nickname, realname, seen_urlpath='./seenurls.pickle'):
		irc.bot.SingleServerIRCBot.__init__(self, [serverspec], nickname, realname)
		self.channel = channel
		self.seen_urlpath = seen_urlpath
		logging.info("nickname=[%s] realname=[%s] channel=[%s]" % (nickname, realname, channel))
		try:
			with open(seen_urlpath, 'rb') as f:
				self.urlhist = pickle.load(f)
		except:
			self.urlhist = {}
		atexit.register(self._dumphist)

	def on_welcome(self, connection, event):
		logging.debug("joining %s", self.channel)
		connection.join(self.channel)

	def _sendmsg(self, connection, msg):
		"""Convenience method to send a msg. Truncates msg to MSG_MAX chars"""
		msg = msg.replace('\n', ' ')
		logging.info("msg: %s" % msg)
		if len(msg) > 140:
			msg = msg[self.MSG_MAX:]
			logging.info("truncated msg: %s" % msg)
		connection.privmsg(self.channel, msg)

	def _dumphist(self):
		try:
			with open(self.seen_urlpath, 'wb') as f:
				pickle.dump(self.urlhist, f)
		except:
			logging.exception("Oh noes, history is lost")

	def urlopen(self, url, maxbytes=FETCH_MAX):
		req = urllib.request.Request(url, headers={'User-agent': 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11'})
		fd = urllib.request.urlopen(req)
		return fd.read(maxbytes)

	def tinify(self, url):
		return self.urlopen('http://tinyurl.com/api-create.php?url=%s' % url)

	def masca(self):
		openers = ('en serio que', 'neta que', 'al chile', '')
		haters = ('me caga', 'me re-caga', 'me molesta un chingo', 'está todo mal', 'me emputa', 'está todo gacho', 'es lo más ojete de la vida', 'es bien tonto')
		closers = ('que fecal fue presidente', 'que no uses C', 'que uses camelCase', 'que existas', 'que un lenguaje no tenga apuntadores', 'que amlo perdió', '')
		standalones = ('Viva el EZLN', '¬¬', '¡Viva la revolución!', 'Estás todo gacho', 'Vamos por cheve')

		composed_len = len(openers) * len(haters) * len(closers)
		totlen = composed_len + len(standalones)

		logging.info('totlen = %d' % totlen)
		logging.info('composed_len = %d' % composed_len)
		# Give equal weight to standalones and composed. This should make standalones a relatively rare occurrence
		if random.randint(0, totlen) < composed_len:
			now = time.clock()
			quote = ' '.join(random.choice(list(itertools.product(openers, haters, closers)))).strip()
			end = time.clock()
			logging.info('Took %d seconds to generate a quote' % ( now - end))
			return quote
		else:
			return random.choice(standalones)

	def rtfm(self, line):
		argv = line.split()
		if len(argv) == 3:
			cmd = argv[2].strip()
			section = argv[1]
		elif len(argv) == 2:
			cmd = argv[1].strip()
			section = None
		else:
			raise Exception("format: !rtfm [section] cmd")

		if not re.match('^[a-zA-Z_-]+$', cmd):
			logging.error("re.match rtfm: %s" % cmd)
			raise Exception("Funky commands not supported")

		if section:
			stdout = subprocess.check_output(['man', '--pager', 'cat', str(section), cmd])
		else:
			stdout = subprocess.check_output(['man', '--pager', 'cat', cmd])
		stdout = stdout.decode('utf-8')

		if stdout:
			seen_description = False
			for line in stdout.splitlines():
				line = line.strip()
				if seen_description:
					return line.split('.')[0]
				if line.endswith('DESCRIPTION'):
					seen_description = True
			raise Exception("Unable to parse manpage")

		raise Exception("No man page found")

	def on_pubmsg(self, connection, event):
		line = event.arguments[0]
		try:
			if line.startswith('!rtfm'):
				self._sendmsg(connection, self.rtfm(line))
			elif line.startswith('!masca'):
				self._sendmsg(connection, self.masca())
			elif line.startswith('ping'):
				self._sendmsg(connection, 'pong')
		except:
			logging.exception("Failed with: %s" % line)
		for u in get_urls(line):
			tinyurl = ''
			repost = ''
			try:
				if u.endswith(('.jpg', '.png', '.git', '.bmp', '.pdf')):
					logging.info('not a webpage, skipping')
					continue
				root = lxml.html.fromstring(self.urlopen(u))
				title = root.find('.//title').text
				if len(u) > self.URL_MAXLEN:
					tinyurl = '[%s]' % self.tinify(u).decode('utf-8')
				if u in self.urlhist:
					repost = '[repost]'
				else:
					self.urlhist[u] = title
				self._sendmsg(connection, '%s %s %s' % (repost, tinyurl, title))
			except:
				logging.exception("Failed with: %s" % line)
				self._sendmsg(connection, random.choice(self.FAIL_MSGS))

def get_args():
	parser = argparse.ArgumentParser()
	parser.add_argument('--server')
	parser.add_argument('--nickname')
	parser.add_argument('--channel')
	parser.add_argument('--realname')
{'fraction_non_alphanumeric': 0.16638257575757576, 'fraction_numerical': 0.010132575757575757, 'mean_word_length': 4.082290664100096, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1704835', 'n_tokens_mistral': 4341, 'n_tokens_neox': 3846, 'n_words': 949}