<commit_before><commit_msg>Initial structure<commit_after>import re
import lxml
import time
import psycopg2

remove_links = re.compile('[A-Za-z0-9')

class page:
	raw = []
	categories = []
	page_title = ""
	def __init__(self,raw,page_title):
		self.raw = raw.split('\n')
		self.page_title = page_title
		#To-Do: Save other relevant data.

	def remove_italics_bold(self):
		more_quotes = True
		for line is self.raw:
			if line.startswith('*') && more_quotes is True:
				line.replace("''","")
				line.replace("'''","")
				print line
				#To-Do: Store in database with psycopg2
			elif line.startswith('== See Also') or line.startswith('==See Also=='):
				more_quotes = False
				continue
			elif line.startswith("[[Category:"):
				#To-Do: Save categories

if __name__ == "__main__":
	quotedb = open("enwikiquote-20140121-pages-articles.xml")
	quotedb = quotedb.read() #Will load entire xml dump into memory. Make sure enough RAM is available.

	##To-Do: Code to parse xml with lxml. Parsing must be done page by page
	##To-Do: Check if redirect tag exists for each page. If it does then store an indication of the same in DB.
	##To-Do: 
{'fraction_non_alphanumeric': 0.13157894736842105, 'fraction_numerical': 0.010526315789473684, 'mean_word_length': 3.9393939393939394, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928205', 'n_tokens_mistral': 428, 'n_tokens_neox': 388, 'n_words': 121}