<commit_before>import ircbot.plugin
import json
import socket
import urllib.error
import urllib.parse
import urllib.request

BASE_URL = 'http://qdb.lutro.me'


def _search_for_quote(quote):
	if isinstance(quote, int):
		url = BASE_URL+'/'+str(quote)
		single = True
	else:
		search = str(quote)
		single = False
		if quote == 'random':
			url = BASE_URL+'/random'
		elif quote == 'latest':
			url = BASE_URL
		else:
			url = BASE_URL+'?'+urllib.parse.urlencode({'s': search})

	try:
		request = urllib.request.Request(url)
		request.add_header('Accept', 'application/json')
		response = urllib.request.urlopen(request, timeout=2)
		content = response.read().decode()
	except (urllib.error.URLError, socket.timeout):
		return 'HTTP error, try again!'

	data = json.loads(content)

	if single:
		quote = data['quote']
	else:
		quotes = data['quotes']
		if 'items' in quotes:
			quotes = quotes['items']
		if len(quotes) < 1:
			return 'No quotes found!'
		quote = quotes[0]

	url = BASE_URL+'/'+str(quote['id'])

	if len(quote['body']) > 160:
		body = quote['body']
		if not single:
			substr_pos = body.lower().index(search.lower())
			start = body.rfind('\n', 0, substr_pos) + 1
			start = max(start, substr_pos - 70)
			end = start + 100 - len(search)
		else:
			start = 0
			end = 100

		body = body.replace('\r', '').replace('\n', ' ').replace('\t', ' ')
		excerpt = body[start:end]

		if start > 0:
			excerpt = '[...] ' + excerpt
		if end < len(quote['body']):
			excerpt = excerpt + ' [...]'
		body = excerpt
	else:
		body = quote['body'].replace('\r', '').replace('\n', ' ').replace('\t', ' ')

	return url + ' - ' + body


class RedditeuQdbPlugin(ircbot.plugin.Plugin):
	@ircbot.plugin.command('qdb')
	def search(self, cmd):
		if len(cmd.args) < 1:
			return

		if cmd.args[0][0] == '#':
			arg = int(cmd.args[0][1:])
		else:
			arg = ' '.join(cmd.args)

		return _search_for_quote(arg)
<commit_msg>make quote search a bit more robust<commit_after>import ircbot.plugin
import json
import socket
import urllib.error
import urllib.parse
import urllib.request

BASE_URL = 'http://qdb.lutro.me'


def _search_for_quote(quote):
	if isinstance(quote, int):
		url = BASE_URL+'/'+str(quote)
		single_quote = True
	else:
		single_quote = False
		search = False
		if quote == 'random':
			url = BASE_URL+'/random'
		elif quote == 'latest':
			url = BASE_URL
		else:
			search = str(quote)
			url = BASE_URL+'?'+urllib.parse.urlencode({'s': search})

	try:
		request = urllib.request.Request(url)
		request.add_header('Accept', 'application/json')
		response = urllib.request.urlopen(request, timeout=2)
		content = response.read().decode()
	except (urllib.error.URLError, socket.timeout):
		return 'HTTP error, try again!'

	data = json.loads(content)

	if single_quote:
		quote = data['quote']
	else:
		quotes = data['quotes']
		if 'items' in quotes:
			quotes = quotes['items']
		if len(quotes) < 1:
			return 'No quotes found!'
		quote = quotes[0]

	url = BASE_URL+'/'+str(quote['id'])

	if len(quote['body']) > 160:
		body = quote['body']
		if search:
			try:
				substr_pos = body.lower().index(search.lower())
				start = body.rfind('\n', 0, substr_pos) + 1
				start = max(start, substr_pos - 70)
				end = start + 100 - len(search)
			except ValueError:
				start = 0
				end = 100
		else:
			start = 0
			end = 100

		body = body.replace('\r', '').replace('\n', ' ').replace('\t', ' ')
		excerpt = body[start:end]

		if start > 0:
			excerpt = '[...] ' + excerpt
		if end < len(quote['body']):
			excerpt = excerpt + ' [...]'
		body = excerpt
	else:
		body = quote['body'].replace('\r', '').replace('\n', ' ').replace('\t', ' ')

	return url + ' - ' + body


class RedditeuQdbPlugin(ircbot.plugin.Plugin):
	@ircbot.plugin.command('qdb')
	def search(self, cmd):
		if len(cmd.args) < 1:
			return

		if cmd.args[0][0] == '#':
			arg = int(cmd.args[0][1:])
		else:
			arg = ' '.join(cmd.args)

		return _search_for_quote(arg)

{'fraction_non_alphanumeric': 0.17989821882951654, 'fraction_numerical': 0.01272264631043257, 'mean_word_length': 3.4925714285714284, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6692801', 'n_tokens_mistral': 1676, 'n_tokens_neox': 1464, 'n_words': 332}