<commit_before>from xml.etree import ElementTree

import requests
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

from apps.curia_vista.models import *
from politkarma import settings


class Command(BaseCommand):
    help = 'Import affair summaries from parlament.ch'

    @transaction.atomic
    def update(self, resource_url, lang, is_main):
        from django.utils import translation
        translation.activate(lang)
        url_template = resource_url + '?format=xml&lang=' + lang + '&pagenumber='
        headers = {'User-Agent': 'Mozilla'}
        cur_page = 1

        while True:
            cur_url = url_template + str(cur_page)
            cur_page += 1
            self.stdout.write("Importing: {}".format(cur_url))

            try:
                response = requests.get(cur_url, headers=headers)
            except Exception as e:
                raise CommandError("Could not fetch file from {}".format(cur_url))

            affair_summaries = ElementTree.fromstring(response.content)

            if not affair_summaries:
                raise CommandError("Not a valid XML file: {}".format(cur_url))

            more_pages = False
            for affair_summary in affair_summaries:
                affair_summery_id = affair_summary.find('id').text
                affair_summery_updated = affair_summary.find('updated').text
                affair_summery_formatted_id = affair_summary.find('formattedId').text
                affair_summery_title = affair_summary.find('title').text
                if affair_summary.find('hasMorePages') is not None:
                    more_pages = 'true' == affair_summary.find('hasMorePages').text
                if is_main:
                    affair_summary_model, created = AffairSummary.objects.update_or_create(id=affair_summery_id,
                                                                                           defaults={
                                                                                               'updated': affair_summery_updated,
                                                                                               'formatted_id': affair_summery_formatted_id,
                                                                                               'title': affair_summery_title})
                else:
                    affair_summary_model, created = AffairSummary.objects.update_or_create(id=affair_summery_id,
                                                                                           updated=affair_summery_updated,
                                                                                           formatted_id=affair_summery_formatted_id,
                                                                                           defaults={
                                                                                               'title': affair_summery_title
                                                                                           })
                    assert not created

                affair_summary_model.full_clean()
                affair_summary_model.save()
                print(affair_summary_model)

            self.stdout.write("Finished importing from {}".format(cur_url))
            if not more_pages:
                break
        self.stdout.write('Done language ' + lang)

    def handle(self, *args, **options):
        is_main = True
        resource_url = 'http://ws.parlament.ch/affairsummaries'
        for lang in [x[0] for x in settings.LANGUAGES]:
            self.update(resource_url, lang, is_main)
            is_main = False<commit_msg>Fix typo<commit_after>from xml.etree import ElementTree

import requests
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

from apps.curia_vista.models import *
from politkarma import settings


class Command(BaseCommand):
    help = 'Import affair summaries from parlament.ch'

    @transaction.atomic
    def update(self, resource_url, lang, is_main):
        from django.utils import translation
        translation.activate(lang)
        url_template = resource_url + '?format=xml&lang=' + lang + '&pagenumber='
        headers = {'User-Agent': 'Mozilla'}
        cur_page = 1

        while True:
            cur_url = url_template + str(cur_page)
            cur_page += 1
            self.stdout.write("Importing: {}".format(cur_url))

            try:
                response = requests.get(cur_url, headers=headers)
            except Exception as e:
                raise CommandError("Could not fetch file from {}".format(cur_url))

            affair_summaries = ElementTree.fromstring(response.content)

            if not affair_summaries:
                raise CommandError("Not a valid XML file: {}".format(cur_url))

            more_pages = False
            for affair_summary in affair_summaries:
                affair_summary_id = affair_summary.find('id').text
                affair_summary_updated = affair_summary.find('updated').text
                affair_summary_formatted_id = affair_summary.find('formattedId').text
                affair_summary_title = affair_summary.find('title').text
                if affair_summary.find('hasMorePages') is not None:
                    more_pages = 'true' == affair_summary.find('hasMorePages').text
                if is_main:
                    affair_summary_model, created = AffairSummary.objects.update_or_create(id=affair_summary_id,
                                                                                           defaults={
                                                                                               'updated': affair_summary_updated,
                                                                                               'formatted_id': affair_summary_formatted_id,
                                                                                               'title': affair_summary_title})
                else:
                    affair_summary_model, created = AffairSummary.objects.update_or_create(id=affair_summary_id,
                                                                                           updated=affair_summary_updated,
                                                                                           formatted_id=affair_summary_formatted_id,
                                                                                           defaults={
                                                                                               'title': affair_summary_title
                                                                                           })
                    assert not created

                affair_summary_model.full_clean()
                affair_summary_model.save()
                print(affair_summary_model)

            self.stdout.write("Finished importing from {}".format(cur_url))
            if not more_pages:
                break
        self.stdout.write('Done language ' + lang)

    def handle(self, *args, **options):
        is_main = True
        resource_url = 'http://ws.parlament.ch/affairsummaries'
        for lang in [x[0] for x in settings.LANGUAGES]:
            self.update(resource_url, lang, is_main)
            is_main = False
{'fraction_non_alphanumeric': 0.06835066864784546, 'fraction_numerical': 0.0008104822369309739, 'mean_word_length': 1.2449969678593087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5395113', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1556, 'n_words': 336}