<commit_before>import re
from io import StringIO

from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage

# TODO: compile regex in this module to improve performance


class BesluitenLijst(object):
    def __init__(self, text):
        self.text = text
        self.title = self.get_title(text)
        self.voortouwcommissie = self.get_voortouwcommissie(text)
        self.activiteitnummer = self.get_activiteitnummer(text)
        self.url = ''
        self.date_published = None
        self.items = create_besluit_items(text)

    def __str__(self):
        return self.title + '\n' + self.voortouwcommissie + '\n' + self.activiteitnummer

    @staticmethod
    def get_title(text):
        pattern = 'Document:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]

    @staticmethod
    def get_voortouwcommissie(text):
        pattern = 'Voortouwcommissie:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]

    @staticmethod
    def get_activiteitnummer(text):
        pattern = 'Activiteitnummer:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]


class BesluitItem(object):
    def __init__(self):
        self.title = ''
        self.cases = []

    def print(self):
        print('Agendapunt: ' + self.title)
        for case in self.cases:
            print('------------')
            case.print()


class BesluitItemCase(object):
    def __init__(self):
        self.title = ''
        self.decisions = []
        self.notes = []
        self.extra = ''
        self.volgcommissies = []

    def print(self):
        print('Zaak: ' + self.title)
        print('decisions:')
        print(self.decisions)
        print('notes:')
        print(self.notes)
        print('volgcommissies:')
        print(self.volgcommissies)


def create_besluit_items(text):
    punten = find_agendapunten(text)
    besluit_items = []
    for punt in punten:
        besluit = BesluitItem()
        besluit.title = punt['title']
        besluit_text = text[punt['start']:punt['end']]
        cases = find_cases(besluit_text)
        for case in cases:
            item_case = BesluitItemCase()
            item_case.title = case['title']
            case_text = besluit_text[case['start']:case['end']]
            decisions = find_decisions(case_text)
            for decision in decisions:
                item_case.decisions.append(decision['title'])
            notes = find_notes(case_text)
            for note in notes:
                item_case.notes.append(note['title'])
            volgcommissies = find_volgcommissies(case_text)
            for volgcommissie in volgcommissies:
                item_case.volgcommissies.append(volgcommissie['title'])
            besluit.cases.append(item_case)
        # besluit.print()
        besluit_items.append(besluit)
    return besluit_items


def besluitenlijst_pdf_to_text(filepath):
    text = pdf_to_text(filepath)
    text = format_text(text)
    # with open('data/lijst.txt', 'w') as fileout:
    #     fileout.write(text)
    return text


def create_besluitenlijst(text):
    obj = BesluitenLijst(text)
    return obj

<commit_msg>get related document ids of besluit case<commit_after>import re
from io import StringIO

from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage

# TODO: compile regex in this module to improve performance


class BesluitenLijst(object):
    def __init__(self, text):
        self.text = text
        self.title = self.get_title(text)
        self.voortouwcommissie = self.get_voortouwcommissie(text)
        self.activiteitnummer = self.get_activiteitnummer(text)
        self.url = ''
        self.date_published = None
        self.items = create_besluit_items(text)

    def __str__(self):
        return self.title + '\n' + self.voortouwcommissie + '\n' + self.activiteitnummer

    @staticmethod
    def get_title(text):
        pattern = 'Document:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]

    @staticmethod
    def get_voortouwcommissie(text):
        pattern = 'Voortouwcommissie:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]

    @staticmethod
    def get_activiteitnummer(text):
        pattern = 'Activiteitnummer:\s+(.*)'
        titles = re.findall(
            pattern=pattern,
            string=text
        )
        return titles[0]


class BesluitItem(object):
    def __init__(self):
        self.title = ''
        self.cases = []

    def print(self):
        print('Agendapunt: ' + self.title)
        for case in self.cases:
            print('------------')
            case.print()


class BesluitItemCase(object):
    def __init__(self, title):
        self.title = title
        self.related_document_ids = self.get_related_document_ids(title)
        self.decisions = []
        self.notes = []
        self.extra = ''
        self.volgcommissies = []

    @staticmethod
    def get_related_document_ids(text):
        pattern = "[0-9]{5}-[0-9]{1,2}"
        document_ids = re.findall(pattern, text)
        return document_ids

    def print(self):
        print('Zaak: ' + self.title)
        print('decisions:')
        print(self.decisions)
        print('notes:')
        print(self.notes)
        print('volgcommissies:')
        print(self.volgcommissies)
        print('related documents:')
        print(self.related_document_ids)


def create_besluit_items(text):
    punten = find_agendapunten(text)
    besluit_items = []
    for punt in punten:
        besluit = BesluitItem()
        besluit.title = punt['title']
        besluit_text = text[punt['start']:punt['end']]
        cases = find_cases(besluit_text)
        for case in cases:
            item_case = BesluitItemCase(case['title'])
            case_text = besluit_text[case['start']:case['end']]
            decisions = find_decisions(case_text)
            for decision in decisions:
                item_case.decisions.append(decision['title'])
            notes = find_notes(case_text)
            for note in notes:
                item_case.notes.append(note['title'])
            volgcommissies = find_volgcommissies(case_text)
            for volgcommissie in volgcommissies:
                item_case.volgcommissies.append(volgcommissie['title'])
            besluit.cases.append(item_case)
        # besluit.print()
        besluit_items.append(besluit)
    return besluit_items


def besluitenlijst_pdf_to_text(filepath):
    text = pdf_to_text(filepath)
    text = format_text(text)
    # with open('data/lijst.txt', 'w') as fileout:
    #     fileout.write(text)
    return text


def create_besluitenlijst(text):
    obj = BesluitenLijst(text)
    return obj


{'fraction_non_alphanumeric': 0.10640066500415628, 'fraction_numerical': 0.0018010529232474369, 'mean_word_length': 2.3145087235996327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4480062', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2282, 'n_words': 419}