<commit_before>from django.core.management.base import LabelCommand, CommandError
from django.utils.encoding import smart_str
from django.db import transaction
from lxml import etree
from lxml.html.builder import P, BR
from os import path
from difflib import ndiff
from optparse import make_option
from itertools import chain
from main.models import User, Document, Transcript, Topic, TopicAssignment

NS = { 'fmp': 'http://www.filemaker.com/fmpxmlresult' }

class Command(LabelCommand):
    args = '<xml_filename xml_filename ...>'
    label = 'XML filename'
    help = 'Imports the specified XML files.'
    option_list = LabelCommand.option_list + (
        make_option('-u',
                    '--username',
                    action='store',
                    default='ryanshaw',
                    help='username that will create/update documents'),
        make_option('-f',
                    '--force-update',
                    action='store_true',
                    default=False,
                    help='update documents whether or not they have changed'),
        )
    def handle_label(self, filename, **options):
        if not path.isfile(filename):
            raise CommandError('%s is not a file.' % filename)
        try:
            doc = etree.parse(filename)
        except etree.LxmlError as e:
            raise CommandError('could not parse %s:\n  %s' % (filename, e))
        root = doc.getroot()
        if root.tag == '{%s}FMPXMLRESULT' % NS['fmp']:
            self.handle_filemaker(doc, **options)
        else:
            raise CommandError('unknown XML schema: %s' % root)

    @transaction.commit_on_success
    def handle_filemaker(self, root, **options):

        # Prefix to append to import IDs.
        ID_PREFIX = 'inglis:'

        # Load user to create/update documents as.
        try:
            user = User.objects.get(username=options['username'])
        except User.DoesNotExist:
            raise CommandError('unknown user: %s' % options['username'])

        # Verify that we have the fields we expect.
        f = expected_fields = [ 'CardID',
                                'Export',
                                'CardHeading',
                                'Transcription',
                                'CardType',
                                'CardFormat',
                                'Language',
                                'ContributorJoin::Contributors',
                                'SubjectJoin::Subject',
                                'OrganizationJoin::Organizations',
                                'Citation',
                                'CatalogLink',
                                'ResourceLink',
                                'AdditionalNotes',
                                'DateEntered' ]
        fields = root.xpath('./fmp:METADATA/fmp:FIELD/@NAME', namespaces=NS)
        if not fields == expected_fields:
            new_fields = [ f for f in fields if not f in expected_fields ]
            missing_fields = [ f for f in expected_fields if not f in fields ]
            message = 'fields have changed:\n'
            message += '\n'.join(ndiff(expected_fields, fields)) + '\n\n'
            if missing_fields:
                message += ('Missing fields:\n  ' + '\n  '.join(missing_fields))
            if new_fields:
                message += ('Unexpected fields:\n  ' + '\n  '.join(new_fields))
            raise CommandError(message)

        # Utility functions for accessing XML data.
        def text(e):
            text = e.text or ''
            for child in e:
                if not child.tag == '{%s}BR' % NS['fmp']:
                    raise CommandError('Unexpected element: %s' % child)
                text += ('\n%s' % (child.tail or ''))
            return text.strip()
        def values(row, field):
            return list(set(
                    [ v for v in [ text(e) for e in row[f.index(field)] ] if v ]))
        def value(row, field):
            v = values(row, field)
            if len(v) == 0:
                return None
            elif len(v) == 1:
                return v[0]
            else:
                raise CommandError('multiple values for %s in record %s'
                                   % (field, row.get('RECORDID')))
        def row_to_dict(row):
            d = {}
            for field in f:
                if 'Join::' in field:
                    d[field] = values(row, field)
                else:
                    d[field] = value(row, field)
            return d

        # Statistics.
        created_count = collections_created_count = topics_created_count = changed_count = unchanged_count = skipped_count = deleted_count = 0

        for row in root.xpath('./fmp:RESULTSET/fmp:ROW', namespaces=NS):
            try:
                md = row_to_dict(row)
                for field in [ 'CardID', 'CardHeading', 'CardType' ]:
                    if md[field] is None:
                        raise CommandError('missing %s value in record %s'
                                           % (field, row.get('RECORDID')))
            except CommandError as e:
                self.stderr.write(self.style.ERROR('Warning: %s\n' % e))
                skipped_count += 1
                continue

            if md['Export'] == 'No':
                exists = Document.objects.filter(
                    import_id__exact='%s%s' % (ID_PREFIX, md['CardID']))<commit_msg>Check if 'Transcription' field is not None in importxml<commit_after>from django.core.management.base import LabelCommand, CommandError
from django.utils.encoding import smart_str
from django.db import transaction
from lxml import etree
from lxml.html.builder import P, BR
from os import path
from difflib import ndiff
from optparse import make_option
from itertools import chain
from main.models import User, Document, Transcript, Topic, TopicAssignment

NS = { 'fmp': 'http://www.filemaker.com/fmpxmlresult' }

class Command(LabelCommand):
    args = '<xml_filename xml_filename ...>'
    label = 'XML filename'
    help = 'Imports the specified XML files.'
    option_list = LabelCommand.option_list + (
        make_option('-u',
                    '--username',
                    action='store',
                    default='ryanshaw',
                    help='username that will create/update documents'),
        make_option('-f',
                    '--force-update',
                    action='store_true',
                    default=False,
                    help='update documents whether or not they have changed'),
        )
    def handle_label(self, filename, **options):
        if not path.isfile(filename):
            raise CommandError('%s is not a file.' % filename)
        try:
            doc = etree.parse(filename)
        except etree.LxmlError as e:
            raise CommandError('could not parse %s:\n  %s' % (filename, e))
        root = doc.getroot()
        if root.tag == '{%s}FMPXMLRESULT' % NS['fmp']:
            self.handle_filemaker(doc, **options)
        else:
            raise CommandError('unknown XML schema: %s' % root)

    @transaction.commit_on_success
    def handle_filemaker(self, root, **options):

        # Prefix to append to import IDs.
        ID_PREFIX = 'inglis:'

        # Load user to create/update documents as.
        try:
            user = User.objects.get(username=options['username'])
        except User.DoesNotExist:
            raise CommandError('unknown user: %s' % options['username'])

        # Verify that we have the fields we expect.
        f = expected_fields = [ 'CardID',
                                'Export',
                                'CardHeading',
                                'Transcription',
                                'CardType',
                                'CardFormat',
                                'Language',
                                'ContributorJoin::Contributors',
                                'SubjectJoin::Subject',
                                'OrganizationJoin::Organizations',
                                'Citation',
                                'CatalogLink',
                                'ResourceLink',
                                'AdditionalNotes',
                                'DateEntered' ]
        fields = root.xpath('./fmp:METADATA/fmp:FIELD/@NAME', namespaces=NS)
        if not fields == expected_fields:
            new_fields = [ f for f in fields if not f in expected_fields ]
            missing_fields = [ f for f in expected_fields if not f in fields ]
            message = 'fields have changed:\n'
            message += '\n'.join(ndiff(expected_fields, fields)) + '\n\n'
            if missing_fields:
                message += ('Missing fields:\n  ' + '\n  '.join(missing_fields))
            if new_fields:
                message += ('Unexpected fields:\n  ' + '\n  '.join(new_fields))
            raise CommandError(message)

        # Utility functions for accessing XML data.
        def text(e):
            text = e.text or ''
            for child in e:
                if not child.tag == '{%s}BR' % NS['fmp']:
                    raise CommandError('Unexpected element: %s' % child)
                text += ('\n%s' % (child.tail or ''))
            return text.strip()
        def values(row, field):
            return list(set(
                    [ v for v in [ text(e) for e in row[f.index(field)] ] if v ]))
        def value(row, field):
            v = values(row, field)
            if len(v) == 0:
                return None
            elif len(v) == 1:
                return v[0]
            else:
                raise CommandError('multiple values for %s in record %s'
                                   % (field, row.get('RECORDID')))
        def row_to_dict(row):
            d = {}
            for field in f:
                if 'Join::' in field:
                    d[field] = values(row, field)
                else:
                    d[field] = value(row, field)
            return d

        # Statistics.
        created_count = collections_created_count = topics_created_count = changed_count = unchanged_count = skipped_count = deleted_count = 0

        for row in root.xpath('./fmp:RESULTSET/fmp:ROW', namespaces=NS):
            try:
                md = row_to_dict(row)
                for field in [ 'CardID', 'CardHeading', 'CardType', 'Transcription']:
                    if md[field] is None:
                        raise CommandError('missing %s value in record %s'
                                           % (field, row.get('RECORDID')))
            except CommandError as e:
                self.stderr.write(self.style.ERROR('Warning: %s\n' % e))
                skipped_count += 1
                continue

            if md['Export'] == 'No':
                exists = Document.objects.filter(
                    import_id__exact='%s%s' % (ID_PREFIX, md['CardID']))
{'fraction_non_alphanumeric': 0.10521993064427815, 'fraction_numerical': 0.0009125752874612155, 'mean_word_length': 1.4810957663572561, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3220292', 'n_tokens_mistral': 2896, 'n_tokens_neox': 2757, 'n_words': 778}