<commit_before># Convert the DC Code .docx file from Lexis into XML.
#
# Usage:
# python3 parse_code_2013-10.py path/to/2013-10.docx

import sys, re, lxml.etree as etree, os.path, json
from worddoc import open_docx

heading_case_fix = { }

def main():
	# Use the West XML to get headings in titlecase. The Lexis document has
	# big level headings in all caps.
	west_dom = etree.parse(open("/home/user/data/dc_code/2012-12-11.xml", "rb"))
	for h in west_dom.xpath('//heading'):
		t = h.text.replace(" (Refs & Annos)", "")
		t = re.sub("[\s\.]+$", "", t)
		heading_case_fix[t.upper()] = t

	# Form the output DOM.
	dom = etree.Element("dc-code")
	meta = make_node(dom, "meta", None)
	make_node(meta, "title", "Code of the District of Columbia")
	make_node(meta, "recency", "current through DC Act 19-658; unofficial through D.C. Act 19-682")
	
	# Open the Word file. Use a cached json file if it exists
	# since that's faster that opening the raw .docx file.
	if not os.path.exists("/tmp/doc.cache.json"):
		doc = open_docx(sys.argv[1], pict=pict_handler)
		with open("/tmp/doc.cache.json", "w") as doccache:
			json.dump(doc, doccache, indent=2)
	else:
		doc = json.load(open("/tmp/doc.cache.json"))
	
	try:
		# Parse each section.
		state = { "stack": None }
		for section in doc["sections"]:
			parse_doc_section(section, dom, state)
	except:
		import traceback
		traceback.print_exc()

	# Output, being careful we get UTF-8 to the byte stream.
	sys.stdout.buffer.write(etree.tostring(dom, pretty_print=True, encoding="utf-8", xml_declaration=True))

def pict_handler(node):
	return "@@PICT@@"

def parse_doc_section(section, dom, state):
	# Parses the Word document, one "section" at a time. By section I mean the things
	# between 'section breaks' in Word. Not code sections.

	sec = None
	annos = None
	anno_levels = None

	for para_index, para in enumerate(section["paragraphs"]):
		psty = para["properties"].get("style")
		if psty in ("sepline",): continue

		ptext = para_text_content(para)
		if ptext.strip() == "": continue

		# Skip everything until the first Division of the Code. There
		# are some divisions before that for constitution things.
		if state["stack"] == None:
			if para["properties"].get("style") != "Division" or not ptext.startswith("DIVISION I"):
				continue
			state["stack"] = [(None, dom)]

		# Correct mistakes in the document.
		context_path = "/".join([n[0] + ":" + n[1].xpath("string(num)") for n in state["stack"][1:]])
		if psty == "sectextc" and ptext in ("PART D-i", "PART A", "PART F-i"):
			# This problem occurs more than once. Oddly specific to "PART D-i".
			psty = "Part"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+2])
			section["paragraphs"][para_index+2]["runs"] = [] # prevent processing later
		elif psty == "sectextc" and re_match("SUBPART \d+A?", ptext):
			psty = "Subpart"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+2])
			section["paragraphs"][para_index+2]["runs"] = [] # prevent processing later
		elif psty == "sectextc" and re_match("CHAPTER 31A1", ptext):
			psty = "Chapter"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+1])
			section["paragraphs"][para_index+1]["runs"] = [] # prevent processing later

		elif psty == "sectextc" and re_match("SUBCHAPTER II-A|BLOOMINGDALE AND LEDROIT PARK BACKWATER VALVES", ptext):
			continue # repeated/weird text
		elif psty == "sectextc" and context_path == "Division:V/Title:32/Chapter:13/Subchapter:I" and ptext in ("SUBCHAPTER I", "GENERAL"):
			continue # repeated/weird text

		elif psty == "Section" and re_match(r"(§ 1-15-\d+\.) \n\n([(\[].*)", ptext):
			# 1-15-1. \n\n(21 DCR 3198; 22 DCR 961....
			ptext = M.group(1)

			# move the rest into the next paragraph
			section["paragraphs"][para_index+1]["properties"]["style"] = "sectext" # make sure it shows up
			section["paragraphs"][para_index+1]["runs"].insert(0, {"text": M.group(2) + "\n", "properties": {}})

		elif psty == "sectext" and re_match("§ [\d\-\.]+\s*-\s*(Appendices to §.*)", ptext):
			# move out of annotations and into an appendices level
			if sec is not None: do_paragraph_indentation(sec)
			sec = make_node(sec, "level", None)
			make_node(sec, "type", "appendices")
			make_node(sec, "heading", M.group(1))
			annos = None

		elif ptext == '(1) A notification of disposition must provide the following information:':
			psty = "sectext" # is Section

		elif ptext.strip() in ("(A)", "(B)") and context_path == "Division:V/Title:29/Chapter:8/Subchapter:IV":
			psty = "analysis"
		elif ptext.strip() in ("(a)", "(b)", "(c)", "(d)", "(1)", "(2)") and context_path == "Division:V/Title:29/Chapter:8/Subchapter:X":
			psty = "analysis"

		elif sec is None and ptext.strip() == "Repealed.":
			psty = "annotations"

		#print(psty, ptext)

		if psty in ("sectext", "sectextc", "form", "formc", "table", "tablec", "PlainText"):
			# "form" appears in §1-204.63.

			if sec is None: raise Exception("Not in a section. %s / %s" % (repr(ptext), context_path))
			if annos is not None: raise Exception("Inside annotations.")

			# Parse paragraph numbering.: If the paragraph starts immediately with a number
			# ("(3)" or "(A)" or a sequence of such numbers) move that into a separate node.
			# Handle numbering like "(3)(A)" by splitting them into multiple nodes. Since the
			# document doesn't encode indentation, we will come back later to nest levels.
			# Also support "1. " numbering, which occurs in the reorg plan sections. Mostly
			# because those also have headings, and it's nicer to parse those headings than
			# to leave them as italic spans.
			is_para_level = False
			parent_node = sec
			while len(para["runs"]) > 0 and re_match("\([0-9A-Za-z\-\.]+\)\s*|[0-9a-z\-]+\.\s+", para["runs"][0]["text"], dollar_sign=False, case_sensitive=True):
				num = M.group(0)
				parent_node = make_node(sec, "level", None)
				make_node(parent_node, "num", num.strip())
				is_para_level = True

				# strip the number we found from the text
				r = para["runs"][0]
				r["text"] = r["text"][len(num):]
				if r["text"] == "": para["runs"].pop(0) # remove run if nothing left

			# Parse italic text at the beginning which is the paragraph's header text.'
			if is_para_level and len(para["runs"]) > 0 and para["runs"][0]["properties"].get("i") == True:
				t = para["runs"][0]["text"]
				t = re.sub("\.?\s*$", "", t) # remove trailing period and whitespace
				make_node(parent_node, "heading", t)
				para["runs"].pop(0) # remove it

			# Make a node for the text in the last numbered-paragraph-level we created.
			if len(para["runs"]) > 0:
				t = make_node(parent_node, "text", "") # initialize with empty text content
				if psty not in ("sectext", "sectextc"): t.set("class", psty)
				runs_to_node(t, para["runs"])

		elif psty in ("annotations", "annotationsc", "history"):
			parent_node = sec
			if sec is None:
				if context_path in (
					"Division:I/Title:8/Subtitle:E/Chapter:21A/Subchapter:II",
					"Division:VI/Title:38/Subtitle:III/Chapter:16"):
					# allow annotations outside of a section
					parent_node = state["stack"][-1][1]
				else:
					raise Exception("Not in a section.")

			if annos is None:
				annos = make_node(parent_node, "level", None)
				make_node(annos, "type", "annotations")
				anno_levels = [(None, annos)]

			if psty == "history":
				level_type = "History"
			elif len(para["runs"]) > 0 and para["runs"][0]["properties"].get("b") == True:
				# initial bold text is an annotation level heading
				level_type = ""
				while len(para["runs"]) > 0 and para["runs"][0]["properties"].get("b") == True:
					level_type += para["runs"][0]["text"]
					para["runs"].pop(0)
				level_type = re.sub("[\.\s]*$", "", level_type) # remove the period and space at the end
				if len(para["runs"]) > 0: para["runs"][0]["text"] = re.sub("^\s*", "", para["runs"][0]["text"]) # remove initial space at the start of the text
			else:<commit_msg>parse sectext paragraph styles into class='centered'<commit_after># Convert the DC Code .docx file from Lexis into XML.
#
# Usage:
# python3 parse_code_2013-10.py path/to/2013-10.docx

import sys, re, lxml.etree as etree, os.path, json
from worddoc import open_docx

heading_case_fix = { }

def main():
	# Use the West XML to get headings in titlecase. The Lexis document has
	# big level headings in all caps.
	west_dom = etree.parse(open("/home/user/data/dc_code/2012-12-11.xml", "rb"))
	for h in west_dom.xpath('//heading'):
		t = h.text.replace(" (Refs & Annos)", "")
		t = re.sub("[\s\.]+$", "", t)
		heading_case_fix[t.upper()] = t

	# Form the output DOM.
	dom = etree.Element("dc-code")
	meta = make_node(dom, "meta", None)
	make_node(meta, "title", "Code of the District of Columbia")
	make_node(meta, "recency", "current through DC Act 19-658; unofficial through D.C. Act 19-682")
	
	# Open the Word file. Use a cached json file if it exists
	# since that's faster that opening the raw .docx file.
	if not os.path.exists("/tmp/doc.cache.json"):
		doc = open_docx(sys.argv[1], pict=pict_handler)
		with open("/tmp/doc.cache.json", "w") as doccache:
			json.dump(doc, doccache, indent=2)
	else:
		doc = json.load(open("/tmp/doc.cache.json"))
	
	try:
		# Parse each section.
		state = { "stack": None }
		for section in doc["sections"]:
			parse_doc_section(section, dom, state)
	except:
		import traceback
		traceback.print_exc()

	# Output, being careful we get UTF-8 to the byte stream.
	sys.stdout.buffer.write(etree.tostring(dom, pretty_print=True, encoding="utf-8", xml_declaration=True))

def pict_handler(node):
	return "@@PICT@@"

def parse_doc_section(section, dom, state):
	# Parses the Word document, one "section" at a time. By section I mean the things
	# between 'section breaks' in Word. Not code sections.

	sec = None
	annos = None
	anno_levels = None

	for para_index, para in enumerate(section["paragraphs"]):
		psty = para["properties"].get("style")
		if psty in ("sepline",): continue

		ptext = para_text_content(para)
		if ptext.strip() == "": continue

		# Skip everything until the first Division of the Code. There
		# are some divisions before that for constitution things.
		if state["stack"] == None:
			if para["properties"].get("style") != "Division" or not ptext.startswith("DIVISION I"):
				continue
			state["stack"] = [(None, dom)]

		# Correct mistakes in the document.
		context_path = "/".join([n[0] + ":" + n[1].xpath("string(num)") for n in state["stack"][1:]])
		if psty == "sectextc" and ptext in ("PART D-i", "PART A", "PART F-i"):
			# This problem occurs more than once. Oddly specific to "PART D-i".
			psty = "Part"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+2])
			section["paragraphs"][para_index+2]["runs"] = [] # prevent processing later
		elif psty == "sectextc" and re_match("SUBPART \d+A?", ptext):
			psty = "Subpart"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+2])
			section["paragraphs"][para_index+2]["runs"] = [] # prevent processing later
		elif psty == "sectextc" and re_match("CHAPTER 31A1", ptext):
			psty = "Chapter"
			ptext += "\n" + para_text_content(section["paragraphs"][para_index+1])
			section["paragraphs"][para_index+1]["runs"] = [] # prevent processing later

		elif psty == "sectextc" and re_match("SUBCHAPTER II-A|BLOOMINGDALE AND LEDROIT PARK BACKWATER VALVES", ptext):
			continue # repeated/weird text
		elif psty == "sectextc" and context_path == "Division:V/Title:32/Chapter:13/Subchapter:I" and ptext in ("SUBCHAPTER I", "GENERAL"):
			continue # repeated/weird text

		elif psty == "Section" and re_match(r"(§ 1-15-\d+\.) \n\n([(\[].*)", ptext):
			# 1-15-1. \n\n(21 DCR 3198; 22 DCR 961....
			ptext = M.group(1)

			# move the rest into the next paragraph
			section["paragraphs"][para_index+1]["properties"]["style"] = "sectext" # make sure it shows up
			section["paragraphs"][para_index+1]["runs"].insert(0, {"text": M.group(2) + "\n", "properties": {}})

		elif psty == "sectext" and re_match("§ [\d\-\.]+\s*-\s*(Appendices to §.*)", ptext):
			# move out of annotations and into an appendices level
			if sec is not None: do_paragraph_indentation(sec)
			sec = make_node(sec, "level", None)
			make_node(sec, "type", "appendices")
			make_node(sec, "heading", M.group(1))
			annos = None

		elif ptext == '(1) A notification of disposition must provide the following information:':
			psty = "sectext" # is Section

		elif ptext.strip() in ("(A)", "(B)") and context_path == "Division:V/Title:29/Chapter:8/Subchapter:IV":
			psty = "analysis"
		elif ptext.strip() in ("(a)", "(b)", "(c)", "(d)", "(1)", "(2)") and context_path == "Division:V/Title:29/Chapter:8/Subchapter:X":
			psty = "analysis"

		elif sec is None and ptext.strip() == "Repealed.":
			psty = "annotations"

		#print(psty, ptext)

		if psty in ("sectext", "sectextc", "form", "formc", "table", "tablec", "PlainText"):
			# "form" appears in §1-204.63.

			if sec is None: raise Exception("Not in a section. %s / %s" % (repr(ptext), context_path))
			if annos is not None: raise Exception("Inside annotations.")

			# Parse paragraph numbering.: If the paragraph starts immediately with a number
			# ("(3)" or "(A)" or a sequence of such numbers) move that into a separate node.
			# Handle numbering like "(3)(A)" by splitting them into multiple nodes. Since the
			# document doesn't encode indentation, we will come back later to nest levels.
			# Also support "1. " numbering, which occurs in the reorg plan sections. Mostly
			# because those also have headings, and it's nicer to parse those headings than
			# to leave them as italic spans.
			is_para_level = False
			parent_node = sec
			while len(para["runs"]) > 0 and re_match("\([0-9A-Za-z\-\.]+\)\s*|[0-9a-z\-]+\.\s+", para["runs"][0]["text"], dollar_sign=False, case_sensitive=True):
				num = M.group(0)
				parent_node = make_node(sec, "level", None)
				make_node(parent_node, "num", num.strip())
				is_para_level = True

				# strip the number we found from the text
				r = para["runs"][0]
				r["text"] = r["text"][len(num):]
				if r["text"] == "": para["runs"].pop(0) # remove run if nothing left

			# Parse italic text at the beginning which is the paragraph's header text.'
			if is_para_level and len(para["runs"]) > 0 and para["runs"][0]["properties"].get("i") == True:
				t = para["runs"][0]["text"]
				t = re.sub("\.?\s*$", "", t) # remove trailing period and whitespace
				make_node(parent_node, "heading", t)
				para["runs"].pop(0) # remove it

			# Make a node for the text in the last numbered-paragraph-level we created.
			if len(para["runs"]) > 0:
				t = make_node(parent_node, "text", "") # initialize with empty text content
				if psty == "sectext":
					pass # no special class
				elif psty == "sectextc":
					t.set("class", "centered")
				else:
					t.set("class", psty)
				runs_to_node(t, para["runs"])

		elif psty in ("annotations", "annotationsc", "history"):
			parent_node = sec
			if sec is None:
				if context_path in (
					"Division:I/Title:8/Subtitle:E/Chapter:21A/Subchapter:II",
					"Division:VI/Title:38/Subtitle:III/Chapter:16"):
					# allow annotations outside of a section
					parent_node = state["stack"][-1][1]
				else:
					raise Exception("Not in a section.")

			if annos is None:
				annos = make_node(parent_node, "level", None)
				make_node(annos, "type", "annotations")
				anno_levels = [(None, annos)]

			if psty == "history":
				level_type = "History"
			elif len(para["runs"]) > 0 and para["runs"][0]["properties"].get("b") == True:
				# initial bold text is an annotation level heading
				level_type = ""
				while len(para["runs"]) > 0 and para["runs"][0]["properties"].get("b") == True:
					level_type += para["runs"][0]["text"]
					para["runs"].pop(0)
				level_type = re.sub("[\.\s]*$", "", level_type) # remove the period and space at the end
				if len(para["runs"]) > 0: para["runs"][0]["text"] = re.sub("^\s*", "", para["runs"][0]["text"]) # remove initial space at the start of the text
			else:
{'fraction_non_alphanumeric': 0.16976773226773226, 'fraction_numerical': 0.01560939060939061, 'mean_word_length': 3.974223602484472, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5861900', 'n_tokens_mistral': 6291, 'n_tokens_neox': 5543, 'n_words': 1629}