<commit_before>import logging
from optparse import make_option
import re
import sys
import urllib2

from django.core.exceptions import (ValidationError, ImproperlyConfigured)
from django.core.management.base import BaseCommand

from lxml import etree

from open511_validator import Open511ValidationError

from open511.models import RoadEvent, Jurisdiction
from open511.utils.serialization import XML_LANG, XML_BASE

logger = logging.getLogger(__name__)


class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('--archive', action='store_true', dest='archive',
            help='Set the status of all events in the jurisdiction *not* in the supplied file to ARCHIVED.'),
    )

    def handle(self, source, **options):
        if re.search(r'^https?://', source):
            source_url = source
            source = urllib2.urlopen(source_url)
        root = etree.parse(source).getroot()
        assert root.tag == 'open511'

        created = []

        opts = {}
        if options['archive']:
            jurisdiction_ids = set(eid.split('/')[0] for eid in root.xpath('events/event/id/text()'))
            if len(jurisdiction_ids) > 1:
                raise ImproperlyConfigured(
                    "To use the archive option, all events must belong to the same jurisdiction.")
            if not jurisdiction_ids:
                raise Exception("Are there events in this file?")
            archive_jurisdiction_id = jurisdiction_ids.pop()

        if root.get(XML_LANG):
            opts['default_language'] = root.get(XML_LANG)

        if root.get(XML_BASE):
            opts['base_url'] = root.get(XML_BASE)

        for event in root.xpath('events/event'):
            try:
                rdev = RoadEvent.objects.update_or_create_from_xml(event, **opts)
                logger.info("Imported event %s" % rdev.id)

                created.append(rdev)

            except (ValueError, ValidationError, Open511ValidationError) as e:
                logger.error("%s importing %s: %s" % (e.__class__.__name__, event.get('id'), e))

        msg = "%s entries imported." % len(created)

        if options['archive']:
            archive_jurisdiction = Jurisdiction.objects.get(id=archive_jurisdiction_id)
            updated = RoadEvent.objects.filter(jurisdiction=archive_jurisdiction).exclude(
                id__in=[rdev.id for rdev in created]).update(active=False)
            msg += " %s events archived." % updated

        print msg<commit_msg>Don't re-archive already archived events in import command<commit_after>import logging
from optparse import make_option
import re
import sys
import urllib2

from django.core.exceptions import (ValidationError, ImproperlyConfigured)
from django.core.management.base import BaseCommand

from lxml import etree

from open511_validator import Open511ValidationError

from open511.models import RoadEvent, Jurisdiction
from open511.utils.serialization import XML_LANG, XML_BASE

logger = logging.getLogger(__name__)


class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('--archive', action='store_true', dest='archive',
            help='Set the status of all events in the jurisdiction *not* in the supplied file to ARCHIVED.'),
    )

    def handle(self, source, **options):
        if re.search(r'^https?://', source):
            source_url = source
            source = urllib2.urlopen(source_url)
        root = etree.parse(source).getroot()
        assert root.tag == 'open511'

        created = []

        opts = {}
        if options['archive']:
            jurisdiction_ids = set(eid.split('/')[0] for eid in root.xpath('events/event/id/text()'))
            if len(jurisdiction_ids) > 1:
                raise ImproperlyConfigured(
                    "To use the archive option, all events must belong to the same jurisdiction.")
            if not jurisdiction_ids:
                raise Exception("Are there events in this file?")
            archive_jurisdiction_id = jurisdiction_ids.pop()

        if root.get(XML_LANG):
            opts['default_language'] = root.get(XML_LANG)

        if root.get(XML_BASE):
            opts['base_url'] = root.get(XML_BASE)

        for event in root.xpath('events/event'):
            try:
                rdev = RoadEvent.objects.update_or_create_from_xml(event, **opts)
                logger.info("Imported event %s" % rdev.id)

                created.append(rdev)

            except (ValueError, ValidationError, Open511ValidationError) as e:
                logger.error("%s importing %s: %s" % (e.__class__.__name__, event.get('id'), e))

        msg = "%s entries imported." % len(created)

        if options['archive']:
            archive_jurisdiction = Jurisdiction.objects.get(id=archive_jurisdiction_id)
            updated = RoadEvent.objects.filter(jurisdiction=archive_jurisdiction, active=True).exclude(
                id__in=[rdev.id for rdev in created]).update(active=False)
            msg += " %s events archived." % updated

        print msg
{'fraction_non_alphanumeric': 0.10256410256410256, 'fraction_numerical': 0.008678500986193294, 'mean_word_length': 2.6429597701149423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '250400', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1375, 'n_words': 361}