<commit_before>import urllib.request, urllib.parse, json, lxml.html

class NoSuchPageException(Exception):
	pass

class DisambiguationPageException(Exception):
	pass

SEARCH_URL = 'http://en.wikiquote.org/w/api.php?format=json&action=query&list=search&continue=&srsearch='
PAGE_URL = 'http://en.wikiquote.org/w/api.php?format=json&action=parse&prop=text|categories&page='
MIN_QUOTE_LEN = 6
MIN_QUOTE_WORDS = 4
DEFAULT_MAX_QUOTES = 20
WORD_BLACKLIST = ['quoted', 'Variant:', 'Retrieved', 'Notes:']

def json_from_url(url):
	res = urllib.request.urlopen(url)
	body = res.read().decode()
	return json.loads(body)

def search(s):
	search_terms = urllib.parse.quote(s)
	data = json_from_url(SEARCH_URL + search_terms)

	results = []
	for entry in data['query']['search']:
		results.append(entry['title'])

	return results

def is_disambiguation(categories):
	if not categories:
		return True
	
	for cat in categories:
		if cat['*'] == 'Disambiguation_pages':
			return True
	return False

def is_quote(txt):
	if not txt or not txt[0].isupper() or len(txt) < MIN_QUOTE_LEN:
		return False

	txt_split = txt.split()
	if len(txt_split) < MIN_QUOTE_WORDS:
		return False

	for word in txt_split:
		if word in WORD_BLACKLIST:
			return False

	if txt.endswith(('(', ':', ']')):
		return False

	return True
	
def extract_quotes(html_content, max_quotes):
	tree = lxml.html.fromstring(html_content)
	quotes_list = []
	
	txt_list = tree.xpath('//ul/li')
	for txt in txt_list:
		txt = txt.text_content().strip()
		if is_quote(txt):
			txt_normal = ' '.join(txt.split())
			quotes_list.append(txt_normal)

			if max_quotes == len(quotes_list):
				break

	return quotes_list
	
def quotes(page_title, max_quotes = DEFAULT_MAX_QUOTES):
	data = json_from_url(PAGE_URL + urllib.parse.quote(page_title))
	if 'error' in data:
		raise NoSuchPageException('No pages matched the title: ' + page_title)

	if is_disambiguation(data['parse']['categories']):
		raise DisambiguationPageException('Title returned a disambiguation page.')

	html_content = data['parse']['text']['*']
	return extract_quotes(html_content, max_quotes)<commit_msg>improved XPath selection path, remove ul subelements from selected nodes<commit_after>import urllib.request, urllib.parse, json, lxml.html

class NoSuchPageException(Exception):
	pass

class DisambiguationPageException(Exception):
	pass

SEARCH_URL = 'http://en.wikiquote.org/w/api.php?format=json&action=query&list=search&continue=&srsearch='
PAGE_URL = 'http://en.wikiquote.org/w/api.php?format=json&action=parse&prop=text|categories&page='
MIN_QUOTE_LEN = 6
MIN_QUOTE_WORDS = 3
DEFAULT_MAX_QUOTES = 20
WORD_BLACKLIST = ['quoted', 'Variant:', 'Retrieved', 'Notes:']

def json_from_url(url):
	res = urllib.request.urlopen(url)
	body = res.read().decode()
	return json.loads(body)

def search(s):
	search_terms = urllib.parse.quote(s)
	data = json_from_url(SEARCH_URL + search_terms)

	results = []
	for entry in data['query']['search']:
		results.append(entry['title'])

	return results

def is_disambiguation(categories):
	if not categories:
		return True

	for cat in categories:
		if cat['*'] == 'Disambiguation_pages':
			return True
	return False

def is_quote(txt):
	if not txt or not txt[0].isupper() or len(txt) < MIN_QUOTE_LEN:
		return False

	txt_split = txt.split()
	if len(txt_split) < MIN_QUOTE_WORDS:
		return False

	for word in txt_split:
		if word in WORD_BLACKLIST:
			return False

	if txt.endswith(('(', ':', ']')):
		return False

	return True

def extract_quotes(html_content, max_quotes):
	tree = lxml.html.fromstring(html_content)
	quotes_list = []

	txt_list = tree.xpath('//div/ul/li')
	for txt in txt_list:

		uls = txt.xpath('ul')
		for ul in uls:
			ul.getparent().remove(ul)

		txt = txt.text_content().strip()
		if is_quote(txt):
			txt_normal = ' '.join(txt.split())
			quotes_list.append(txt_normal)

			if max_quotes == len(quotes_list):
				break

	return quotes_list

def quotes(page_title, max_quotes = DEFAULT_MAX_QUOTES):
	data = json_from_url(PAGE_URL + urllib.parse.quote(page_title))
	if 'error' in data:
		raise NoSuchPageException('No pages matched the title: ' + page_title)

	if is_disambiguation(data['parse']['categories']):
		raise DisambiguationPageException('Title returned a disambiguation page.')

	html_content = data['parse']['text']['*']
	return extract_quotes(html_content, max_quotes)
{'fraction_non_alphanumeric': 0.13288960293780125, 'fraction_numerical': 0.0022951572182694515, 'mean_word_length': 5.380673499267935, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4459349', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1542, 'n_words': 333}