<commit_before># -*- coding: utf-8 -*-
from optparse import make_option
import os
import re
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.core.management.base import LabelCommand, CommandError, BaseCommand
from django.core.files import File

from lxml import etree, html
from os import path

from haystack.backends.solr_backend import SolrSearchBackend
from pysolr import SolrError
from open_municipio.people.models import Person, municipality
from open_municipio.acts.models import *

import logging

from open_municipio.data_import import conf

# configure xml namespaces
NS = {
    'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
    'om': 'http://www.openmunicipio.it',
    'xlink': 'http://www.w3.org/1999/xlink'
}
XLINK_NAMESPACE = NS['xlink']
XLINK = "{%s}" % XLINK_NAMESPACE

class ImportActsCommand(LabelCommand):
    option_list = BaseCommand.option_list + (
        make_option('--people-file',
                    dest='people_file',
                    default=conf.ACTS_PEOPLE_FILE,
                    help='The xml file containing the persons ids mappings'
        ),
        make_option('--act-type',
                    dest='act_type',
                    default='CouncilDeliberation',
                    help='The type of act to import (CouncilDeliberation, Motion, Interrogation)'
        ),
        make_option('--dry-run',
                    action='store_true',
                    dest='dry_run',
                    default=False,
                    help='Execute without actually writing into the DB'
        ),
        make_option('--refresh-news',
                    action='store_true',
                    dest='refresh_news',
                    default=False,
                    help='Remove news related to an act, before importing it. News are re-created.'
        ),
        make_option('--rewrite-signatures',
                    action='store_true',
                    dest='rewrite_signatures',
                    default=False,
                    help="Remove act's presenters before importing."
        ),
    )

    args = "<filename filename ...>"
    help = "Import the act(s) of type act_type, contained in the specified XML document(s)."
    label = 'filename'

    dry_run = False

    logger = logging.getLogger('import')

    people_tree = None

    def lookupCharge(self, xml_chargexref, institution=None, moment=None):
        """
        look for the correct open municipio charge, or return None
        if the date parameter is not passed, then current charges are looked up
        """
        try:
            file, charge_id = xml_chargexref.get(XLINK+"href").split("#")
            people_charges = self.people_tree.xpath(
                '//om:Person[@id="%s"]' % charge_id,
                namespaces=NS
            )
            if len(people_charges):
                om_id = people_charges[0].get('om_id')
                if om_id is None:
                    self.logger.warning("charge with id %s has no om_id (past charge?). Skipping." % charge_id)
                    return None
                charge_type = people_charges[0].get('charge')
                if charge_type is None:
                    self.logger.warning("charge with id %s has no charge attribute. Skipping." % charge_id)
                    return None

                # institution is grabbed from charge attribute, in acts import
                # since mayor and deputies may sign acts, not only counselor
                if charge_type == 'counselor':
                    institution = municipality.council.as_institution
                elif charge_type == 'deputy' or charge_type == 'firstdeputy':
                    institution = municipality.gov.as_institution
                elif charge_type == 'mayor':
                    institution = municipality.mayor.as_institution
                else:
                    self.logger.error("Warning: charge with id %s has wrong charge attribute %s. Skipping." %
                                      (charge_id, charge_type))
                    return None

                try:
                    person = Person.objects.get(pk=int(om_id))
                    charge =  person.current_institution_charge(institution, moment=moment)
                    self.logger.debug("id %s (%s) mapped to %s (%s)" %
                                      (charge_id, charge_type, person, charge))
                    return charge
                except ObjectDoesNotExist:
                    self.logger.warning("could not find person or charge for id = %s in open municipio DB. Skipping." % charge_id)
                    return None
                except MultipleObjectsReturned:
                    self.logger.error("found more than one person or charge for id %s in open municipio db. Skipping." % charge_id)
                    return None
            else:
                self.logger.warning("could not find person for id %s in people XML file. Skipping." % charge_id)
                return None
        except ObjectDoesNotExist:
            self.logger.warning("could not find charge for %s in Open Municipio DB. Skipping." % xml_chargexref)
            return None

    def fetch_signers(self, om_act, xml_subscribers_set, support_type, charge_lookup_institution):
        """
        extract signers from an om:ActSubscribers tag and build related ActSupport records<commit_msg>ImportVotationsCommand stub implemented; the real task is implemented in the om-x instance command task<commit_after># -*- coding: utf-8 -*-
from optparse import make_option
import os
import re
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.core.management.base import LabelCommand, CommandError, BaseCommand
from django.core.files import File

from lxml import etree, html
from os import path

from haystack.backends.solr_backend import SolrSearchBackend
from pysolr import SolrError
from open_municipio.people.models import Person, municipality
from open_municipio.acts.models import *

import logging

from open_municipio.data_import import conf

# configure xml namespaces
NS = {
    'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
    'om': 'http://www.openmunicipio.it',
    'xlink': 'http://www.w3.org/1999/xlink'
}
XLINK_NAMESPACE = NS['xlink']
XLINK = "{%s}" % XLINK_NAMESPACE

class ImportVotationsCommand(LabelCommand):
    option_list = BaseCommand.option_list + (
        make_option('--people-file',
                    dest='people_file',
                    default="people.xml",
                    help='The xml file containing the persons ids mappings'
        ),
        make_option('--dry-run',
                    action='store_true',
                    dest='dry_run',
                    default=False,
                    help='Execute without actually writing into the DB'
        ),
    )

    args = "<filename filename ...>"
    help = "Import the votation(s) contained in the specified MDB file(s)."
    label = 'filename'

    dry_run = False

    logger = logging.getLogger('import')

    people_tree = None

    def handle_label(self, filename, **options):
        raise Exception("Not implemented")

    def handle(self, *labels, **options):
        if not labels:
            raise CommandError('Enter at least one %s.' % self.label)

        # parse people xml file into an lxml.etree
        people_file = options['people_file']
        if not path.isfile(people_file):
            raise IOError("File %s does not exist" % people_file)

        self.people_tree = etree.parse(people_file)

        self.dry_run = options['dry_run']

        # parse passed votations
        for label in labels:
            self.handle_label(label, **options)
        return 'done'


class ImportActsCommand(LabelCommand):
    option_list = BaseCommand.option_list + (
        make_option('--people-file',
                    dest='people_file',
                    default=conf.ACTS_PEOPLE_FILE,
                    help='The xml file containing the persons ids mappings'
        ),
        make_option('--act-type',
                    dest='act_type',
                    default='CouncilDeliberation',
                    help='The type of act to import (CouncilDeliberation, Motion, Interrogation)'
        ),
        make_option('--dry-run',
                    action='store_true',
                    dest='dry_run',
                    default=False,
                    help='Execute without actually writing into the DB'
        ),
        make_option('--refresh-news',
                    action='store_true',
                    dest='refresh_news',
                    default=False,
                    help='Remove news related to an act, before importing it. News are re-created.'
        ),
        make_option('--rewrite-signatures',
                    action='store_true',
                    dest='rewrite_signatures',
                    default=False,
                    help="Remove act's presenters before importing."
        ),
    )

    args = "<filename filename ...>"
    help = "Import the act(s) of type act_type, contained in the specified XML document(s)."
    label = 'filename'

    dry_run = False

    logger = logging.getLogger('import')

    people_tree = None

    def lookupCharge(self, xml_chargexref, institution=None, moment=None):
        """
        look for the correct open municipio charge, or return None
        if the date parameter is not passed, then current charges are looked up
        """
        try:
            file, charge_id = xml_chargexref.get(XLINK+"href").split("#")
            people_charges = self.people_tree.xpath(
                '//om:Person[@id="%s"]' % charge_id,
                namespaces=NS
            )
            if len(people_charges):
                om_id = people_charges[0].get('om_id')
                if om_id is None:
                    self.logger.warning("charge with id %s has no om_id (past charge?). Skipping." % charge_id)
                    return None
                charge_type = people_charges[0].get('charge')
                if charge_type is None:
                    self.logger.warning("charge with id %s has no charge attribute. Skipping." % charge_id)
                    return None

                # institution is grabbed from charge attribute, in acts import
                # since mayor and deputies may sign acts, not only counselor
                if charge_type == 'counselor':
                    institution = municipality.council.as_institution
                elif charge_type == 'deputy' or charge_type == 'firstdeputy':
                    institution = municipality.gov.as_institution
                elif charge_type == 'mayor':
                    institution = municipality.mayor.as_institution
                else:
                    self.logger.error("Warning: charge with id %s has wrong charge attribute %s. Skipping." %
                                      (charge_id, charge_type))
                    return None

                try:
                    person = Person.objects.get(pk=int(om_id))
                    charge =  person.get_current_charge_in_institution(institution, moment=moment)
                    self.logger.debug("id %s (%s) mapped to %s (%s)" %
                                      (charge_id, charge_type, person, charge))
                    return charge
                except ObjectDoesNotExist:
                    self.logger.warning("could not find person or charge for id = %s in open municipio DB. Skipping." % charge_id)
                    return None
                except MultipleObjectsReturned:
                    self.logger.error("found more than one person or charge for id %s in open municipio db. Skipping." % charge_id)
                    return None
            else:
                self.logger.warning("could not find person for id %s in people XML file. Skipping." % charge_id)
                return None
        except ObjectDoesNotExist:
            self.logger.warning("could not find charge for %s in Open Municipio DB. Skipping." % xml_chargexref)
            return None

    def fetch_signers(self, om_act, xml_subscribers_set, support_type, charge_lookup_institution):
        """
        extract signers from an om:ActSubscribers tag and build related ActSupport records
{'fraction_non_alphanumeric': 0.08774339500686758, 'fraction_numerical': 0.0021006705986911208, 'mean_word_length': 1.8786046511627907, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2350610', 'n_tokens_mistral': 3294, 'n_tokens_neox': 3093, 'n_words': 904}