<commit_before>import datetime
import logging
import time
import multiprocessing as mp

from requests.exceptions import ConnectionError, ConnectTimeout

from django.db import transaction

from tkapi import Api
from tkapi.zaak import Zaak

import scraper.documents
import scraper.dossiers

from document.models import CategoryDossier
from document.models import Dossier
from document.models import Kamerstuk

from openkamer.agenda import create_agenda
from openkamer.document import update_document_html_links
from openkamer.document import DocumentFactory
from openkamer.document import DocumentData
from openkamer.document import get_categories
from openkamer.kamerstuk import create_kamerstuk
from openkamer.voting import VotingFactory

logger = logging.getLogger(__name__)


def create_dossier_retry_on_error(dossier_id, max_tries=3):
    dossier_id = str(dossier_id)
    tries = 0
    while True:
        try:
            tries += 1
            create_or_update_dossier(dossier_id)
        except (ConnectionError, ConnectTimeout) as error:
            logger.exception(error)
            time.sleep(5)  # wait 5 seconds for external servers to relax
            if tries < max_tries:
                logger.error('trying again!')
                continue
            logger.error('max tries reached, skipping dossier: ' + dossier_id)
        break


@transaction.atomic
def create_or_update_dossier(dossier_id):
    logger.info('BEGIN - dossier id: ' + str(dossier_id))
    Dossier.objects.filter(dossier_id=dossier_id).delete()
    dossier_url = scraper.dossiers.search_dossier_url(dossier_id)
    last_besluit = get_besluit_last(dossier_id)
    decision = 'Onbekend'
    if last_besluit:
        decision = last_besluit.slottekst.replace('.', '')
    dossier_new = Dossier.objects.create(
        dossier_id=dossier_id,
        url=dossier_url,
        decision=decision
    )
    create_dossier_documents(dossier_new, dossier_id)
    voting_factory = VotingFactory()
    voting_factory.create_votings(dossier_id)
    dossier_new.set_derived_fields()
    logger.info('END - dossier id: ' + str(dossier_id))
    return dossier_new


class DocumentDataPolitiekNL(object):

    def __init__(self, search_result, document_id, title, metadata, content_html):
        self.search_result = search_result
        self.document_id = document_id
        self.title = title
        self.metadata = metadata
        self.content_html = content_html
        self.url = search_result['page_url']


def get_document_data_mp(search_result, outputs):
    # skip eerste kamer documents, first focus on the tweede kamer
    # TODO: handle eerste kamer documents
    if 'eerste kamer' in search_result['publisher'].lower():
        logger.info('skipping Eerste Kamer document')
        return
    # skip documents of some types and/or sources, no models implemented yet
    # TODO: handle all document types
    if 'Staatscourant' in search_result['type']:
        logger.info('Staatscourant, skip for now')
        return

    if 'Agenda' in search_result['type']:
        logger.info('Agenda, skip for now')
        return

    document_id, content_html, title = scraper.documents.get_document_id_and_content(search_result['page_url'])
    if not document_id:
        logger.error('No document id found for url: ' + search_result['page_url'] + ' - will not create document')
        return

    metadata = scraper.documents.get_metadata(document_id)
    document_data = DocumentDataPolitiekNL(
        search_result=search_result,
        document_id=document_id,
        title=title,
        metadata=metadata,
        content_html=content_html
    )
    outputs.append(document_data)


@transaction.atomic
def create_dossier_documents(dossier, dossier_id):
    search_results = scraper.documents.search_politieknl_dossier(dossier_id)

    pool = mp.Pool()
    manager = mp.Manager()
    outputs = manager.list()
    for search_result in search_results:
        pool.apply_async(get_document_data_mp, args=(search_result, outputs))
    pool.close()
    pool.join()

    for data in outputs:
        if data.metadata['date_published']:
            data.metadata['date_published'] = data.metadata['date_published']
        else:
            data.metadata['date_published'] = data.search_result['date_published']

        if 'submitter' not in data.metadata:
            data.metadata['submitter'] = 'undefined'

        dossier_for_document = dossier<commit_msg>Replace mp Pool with ThreadPool<commit_after>import datetime
import logging
import time
import multiprocessing as mp
from multiprocessing.pool import ThreadPool

from requests.exceptions import ConnectionError, ConnectTimeout

from django.db import transaction

from tkapi import Api
from tkapi.zaak import Zaak

import scraper.documents
import scraper.dossiers

from document.models import CategoryDossier
from document.models import Dossier
from document.models import Kamerstuk

from openkamer.agenda import create_agenda
from openkamer.document import update_document_html_links
from openkamer.document import DocumentFactory
from openkamer.document import DocumentData
from openkamer.document import get_categories
from openkamer.kamerstuk import create_kamerstuk
from openkamer.voting import VotingFactory

logger = logging.getLogger(__name__)


def create_dossier_retry_on_error(dossier_id, max_tries=3):
    dossier_id = str(dossier_id)
    tries = 0
    while True:
        try:
            tries += 1
            create_or_update_dossier(dossier_id)
        except (ConnectionError, ConnectTimeout) as error:
            logger.exception(error)
            time.sleep(5)  # wait 5 seconds for external servers to relax
            if tries < max_tries:
                logger.error('trying again!')
                continue
            logger.error('max tries reached, skipping dossier: ' + dossier_id)
        break


@transaction.atomic
def create_or_update_dossier(dossier_id):
    logger.info('BEGIN - dossier id: ' + str(dossier_id))
    Dossier.objects.filter(dossier_id=dossier_id).delete()
    dossier_url = scraper.dossiers.search_dossier_url(dossier_id)
    last_besluit = get_besluit_last(dossier_id)
    decision = 'Onbekend'
    if last_besluit:
        decision = last_besluit.slottekst.replace('.', '')
    dossier_new = Dossier.objects.create(
        dossier_id=dossier_id,
        url=dossier_url,
        decision=decision
    )
    create_dossier_documents(dossier_new, dossier_id)
    voting_factory = VotingFactory()
    voting_factory.create_votings(dossier_id)
    dossier_new.set_derived_fields()
    logger.info('END - dossier id: ' + str(dossier_id))
    return dossier_new


class DocumentDataPolitiekNL(object):

    def __init__(self, search_result, document_id, title, metadata, content_html):
        self.search_result = search_result
        self.document_id = document_id
        self.title = title
        self.metadata = metadata
        self.content_html = content_html
        self.url = search_result['page_url']


def get_document_data_mp(search_result, outputs):
    # skip eerste kamer documents, first focus on the tweede kamer
    # TODO: handle eerste kamer documents
    if 'eerste kamer' in search_result['publisher'].lower():
        logger.info('skipping Eerste Kamer document')
        return
    # skip documents of some types and/or sources, no models implemented yet
    # TODO: handle all document types
    if 'Staatscourant' in search_result['type']:
        logger.info('Staatscourant, skip for now')
        return

    if 'Agenda' in search_result['type']:
        logger.info('Agenda, skip for now')
        return

    document_id, content_html, title = scraper.documents.get_document_id_and_content(search_result['page_url'])
    if not document_id:
        logger.error('No document id found for url: ' + search_result['page_url'] + ' - will not create document')
        return

    metadata = scraper.documents.get_metadata(document_id)
    document_data = DocumentDataPolitiekNL(
        search_result=search_result,
        document_id=document_id,
        title=title,
        metadata=metadata,
        content_html=content_html
    )
    outputs.append(document_data)


@transaction.atomic
def create_dossier_documents(dossier, dossier_id):
    search_results = scraper.documents.search_politieknl_dossier(dossier_id)

    pool = ThreadPool()
    manager = mp.Manager()
    outputs = manager.list()
    for search_result in search_results:
        pool.apply_async(get_document_data_mp, args=(search_result, outputs))
    pool.close()
    pool.join()

    for data in outputs:
        if data.metadata['date_published']:
            data.metadata['date_published'] = data.metadata['date_published']
        else:
            data.metadata['date_published'] = data.search_result['date_published']

        if 'submitter' not in data.metadata:
            data.metadata['submitter'] = 'undefined'

        dossier_for_document = dossier
{'fraction_non_alphanumeric': 0.0785276073619632, 'fraction_numerical': 0.0011154489682097045, 'mean_word_length': 3.503264691109995, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4399463', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2625, 'n_words': 611}