<commit_before>#!/usr/bin/env python3

import decimal
import re
import ezodf
import tableWriters

def makeTableReaderFromOdfTable(table):
	def reader(years):
		found=False
		for row in obj.rows():
			if not found:
				if row[0].value.strip()=='1.':
					found=True
				elif row[0].value.strip()=='.1.': # quirk in 3765.10.3
					found=True
				else:
					continue
			keys=(row[0].value.strip(),row[1].value.strip(),row[2].value.strip()+row[3].value.strip(),row[4].value.strip(),row[5].value.strip())
			def makeAmount(cell):
				amount=cell.value.strip()
				if '.' not in amount:
					# quirk in 3765.7.1.1 and 3765.7.5.1
					amount=amount[:-1]+'.'+amount[-1]
				return decimal.Decimal(amount)
			amounts=[makeAmount(row[i]) for i in range(6,6+len(years))]
			yield (keys,amounts)
		if not found:
			raise Exception('table start not found')
	return reader

class TableWriteWatcher:
	def __init__(self,years):
		self.years=years
		self.paragraphNumber=None
	def setParagraphNumber(self,paragraphNumber):
		self.paragraphNumber=paragraphNumber
	def write(self,table,amendmentNumber):
		# print('writing table for amendment',self.paragraphNumber)
		if not self.paragraphNumber:
			raise Exception('paragraph number for table not set')
		tableWriters.DepartmentTableWriter(
			makeTableReaderFromOdfTable(table),
                        amendmentNumber,self.years
		).write('tables/3765.'+self.paragraphNumber+'csv')
		self.paragraphNumber=None

# paragraphRe=re.compile(r'Пункт N (?P<paragraphNumber>\d+(?:\.\d+)*)')
amendParagraphTextRe=re.compile(r'(?P<paragraphNumber>(?:\d+\.)+) В текстовую часть')
amendParagraphAppendixRe=re.compile(r'(?P<paragraphNumber>(?:\d+\.)+) В приложение (?P<appendixNumber>\d+)')
amendAppendixRe=re.compile(r'В приложение (?P<appendixNumber>\d+)')
subParagraphRe=re.compile(r'\s+(?P<paragraphNumber>(?:\d+\.){2,})')

doc=ezodf.opendoc('assembly/3765.odt')
tableWriteWatcher=None
amendmentNumber=0
for obj in doc.body:
	if type(obj) is ezodf.text.Paragraph:
		# print('paragraph {')
		for line in obj.plaintext().splitlines():
			# m=paragraphRe.match(line)
			# if m:
				# print('== paragraph',m.group('paragraphNumber'),'==')
			m=amendParagraphTextRe.match(line)
			if m:
				# print('== amendment',m.group('paragraphNumber'),'for text ==')
				tableWriteWatcher=None
			m=amendParagraphAppendixRe.match(line)
			if m:
				# print('== amendment',m.group('paragraphNumber'),'for appendix',m.group('appendixNumber'),'==')
				if m.group('appendixNumber') in ('3','4'):
					tableWriteWatcher=TableWriteWatcher([2014] if m.group('appendixNumber')=='3' else [2015,2016])
					tableWriteWatcher.setParagraphNumber(m.group('paragraphNumber'))
				else:
					tableWriteWatcher=None
			m=amendAppendixRe.match(line)
			if m:
				# print('== amendment TBD for appendix',m.group('appendixNumber'),'==')
				if m.group('appendixNumber') in ('3','4'):
					tableWriteWatcher=TableWriteWatcher([2014] if m.group('appendixNumber')=='3' else [2015,2016])
				else:
					tableWriteWatcher=None
			m=subParagraphRe.match(line)
			if m:
				# print('=== amendment',m.group('paragraphNumber'),'===')
				if tableWriteWatcher:
					tableWriteWatcher.setParagraphNumber(m.group('paragraphNumber'))
			# print('line:',line)
		# print('}')
	elif type(obj) is ezodf.table.Table:
		# print('table',obj.nrows(),'x',obj.ncols())
		if tableWriteWatcher:
			amendmentNumber+=1
			tableWriteWatcher.write(obj,amendmentNumber)<commit_msg>document 3781<commit_after>#!/usr/bin/env python3

import decimal
import re
import ezodf
import tableWriters

def makeTableReaderFromOdfTable(table):
	def reader(years):
		found=False
		for row in obj.rows():
			if not found:
				if row[0].value.strip()=='1.':
					found=True
				elif row[0].value.strip()=='.1.': # quirk in 3765.10.3
					found=True
				else:
					continue
			keys=(row[0].value.strip(),row[1].value.strip(),row[2].value.strip()+row[3].value.strip(),row[4].value.strip(),row[5].value.strip())
			def makeAmount(cell):
				amount=cell.value.strip()
				if '.' not in amount:
					# quirk in 3765.7.1.1 and 3765.7.5.1
					amount=amount[:-1]+'.'+amount[-1]
				elif amount.count('.')==2:
					# quirk in 3781.2.34.7.1.
					amount=amount.replace('.','',1)
				return decimal.Decimal(amount)
			amounts=[makeAmount(row[i]) for i in range(6,6+len(years))]
			yield (keys,amounts)
		if not found:
			raise Exception('table start not found')
	return reader

class TableWriteWatcher:
	def __init__(self,years):
		self.years=years
		self.paragraphNumber=None
	def setParagraphNumber(self,paragraphNumber):
		self.paragraphNumber=paragraphNumber
	def write(self,table,documentNumber,amendmentNumber):
		# print('writing table for amendment',self.paragraphNumber)
		if not self.paragraphNumber:
			raise Exception('paragraph number for table not set')
		tableWriters.DepartmentTableWriter(
			makeTableReaderFromOdfTable(table),
                        amendmentNumber,self.years
		).write('tables/'+documentNumber+'.'+self.paragraphNumber+'csv')
		self.paragraphNumber=None

# paragraphRe=re.compile(r'Пункт N (?P<paragraphNumber>\d+(?:\.\d+)*)')
amendParagraphTextRe=re.compile(r'(?P<paragraphNumber>(?:\d+\.)+) В текстовую часть')
amendParagraphAppendixRe=re.compile(r'(?P<paragraphNumber>(?:\d+\.)+) В приложение (?P<appendixNumber>\d+)')
amendAppendixRe=re.compile(r'В приложение (?P<appendixNumber>\d+)')
subParagraphRe=re.compile(r'\s+(?P<paragraphNumber>(?:\d+\.){2,})')

amendmentNumber=0
for documentNumber in ('3765','3781'):
	filename='assembly/'+documentNumber+'.odt'
	doc=ezodf.opendoc(filename)
	tableWriteWatcher=None
	for obj in doc.body:
		if type(obj) is ezodf.text.Paragraph:
			# print('paragraph {')
			for line in obj.plaintext().splitlines():
				# m=paragraphRe.match(line)
				# if m:
					# print('== paragraph',m.group('paragraphNumber'),'==')
				m=amendParagraphTextRe.match(line)
				if m:
					# print('== amendment',m.group('paragraphNumber'),'for text ==')
					tableWriteWatcher=None
				m=amendParagraphAppendixRe.match(line)
				if m:
					# print('== amendment',m.group('paragraphNumber'),'for appendix',m.group('appendixNumber'),'==')
					if m.group('appendixNumber') in ('3','4'):
						tableWriteWatcher=TableWriteWatcher([2014] if m.group('appendixNumber')=='3' else [2015,2016])
						tableWriteWatcher.setParagraphNumber(m.group('paragraphNumber'))
					else:
						tableWriteWatcher=None
				m=amendAppendixRe.match(line)
				if m:
					# print('== amendment TBD for appendix',m.group('appendixNumber'),'==')
					if m.group('appendixNumber') in ('3','4'):
						tableWriteWatcher=TableWriteWatcher([2014] if m.group('appendixNumber')=='3' else [2015,2016])
					else:
						tableWriteWatcher=None
				m=subParagraphRe.match(line)
				if m:
					# print('=== amendment',m.group('paragraphNumber'),'===')
					if tableWriteWatcher:
						tableWriteWatcher.setParagraphNumber(m.group('paragraphNumber'))
				# print('line:',line)
			# print('}')
		elif type(obj) is ezodf.table.Table:
			# print('table',obj.nrows(),'x',obj.ncols())
			if tableWriteWatcher:
				amendmentNumber+=1
				tableWriteWatcher.write(obj,documentNumber,amendmentNumber)
{'fraction_non_alphanumeric': 0.17271326516318813, 'fraction_numerical': 0.02367278330298361, 'mean_word_length': 4.871710526315789, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3473056', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2440, 'n_words': 401}