<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from flask_pluginengine import current_plugin

from MaKaC.accessControl import AccessWrapper, AdminList
from MaKaC.common.output import outputGenerator
from MaKaC.common.xmlGen import XMLGen

from indico_livesync import process_records, SimpleChange
from indico_livesync.util import make_compound_id, obj_deref


class MARCXMLGenerator:
    """Generates MARCXML based on Indico objects"""

    @classmethod
    def records_to_xml(cls, records):
        mg = MARCXMLGenerator()
        for ref, change in process_records(records).iteritems():
            try:
                mg.add_object(ref, bool(change & SimpleChange.deleted))
            except Exception:
                mg._remove_unfinished_record()
                current_plugin.logger.exception('Could not process {}'.format(ref))
        return mg.get_xml()

    def __init__(self):
        self.closed = False
        self.xml_generator = XMLGen()
        self.xml_generator.initXml()
        self.xml_generator.openTag('collection', [['xmlns', 'http://www.loc.gov/MARC21/slim']])
        # This is horrible. but refactoring all the code in the indico core would be just as bad.
        aw = AccessWrapper()
        aw.setUser(AdminList().getInstance().getList()[0])
        self.output_generator = outputGenerator(aw, self.xml_generator)

    def add_object(self, ref, deleted=False):
        if self.closed:
            raise RuntimeError('Cannot add object to closed xml generator')
        if deleted:
            self.xml_generator.openTag('record')

            self.xml_generator.openTag('datafield', [['tag', '970'], ['ind1', ' '], ['ind2', ' ']])
            self.xml_generator.writeTag('subfield', 'INDICO.{}'.format(make_compound_id(ref)), [['code', 'a']])
            self.xml_generator.closeTag('datafield')

            self.xml_generator.openTag('datafield', [['tag', '980'], ['ind1', ' '], ['ind2', ' ']])
            self.xml_generator.writeTag('subfield', 'DELETED', [['code', 'c']])
            self.xml_generator.closeTag('datafield')

            self.xml_generator.closeTag('record')
        elif ref['type'] in {'event', 'contribution', 'subcontribution'}:
            obj = obj_deref(ref)
            if obj is None:
                raise ValueError('Cannot add deleted object')
            elif not obj.getOwner():
                raise ValueError('Cannot add object without owner: {}'.format(obj))
            if ref['type'] == 'event':
                self._event_to_marcxml(obj)
            elif ref['type'] == 'contribution':
                self._contrib_to_marcxml(obj)
            elif ref['type'] == 'subcontribution':
                self._subcontrib_to_marcxml(obj)
        else:
            raise ValueError('unknown object ref: {}'.format(ref['type']))
        return self.xml_generator.getXml()

    def get_xml(self):
        if not self.closed:
            self.xml_generator.closeTag('collection')
        return self.xml_generator.getXml()

    def _remove_unfinished_record(self):
        # remove any line breaks, etc...
        while not self.xml_generator.xml[-1].strip():
            self.xml_generator.xml.pop()
        # remove '<record>'
        self.xml_generator.xml.pop()

    def _event_to_marcxml(self, obj):
        self.xml_generator.openTag('record')
        self.output_generator.confToXMLMarc21(obj, out=self.xml_generator, overrideCache=True)
        self.xml_generator.closeTag('record')

    def _contrib_to_marcxml(self, obj):
        self.xml_generator.openTag('record')
        self.output_generator.contribToXMLMarc21(obj, out=self.xml_generator, overrideCache=True)
        self.xml_generator.closeTag('record')

    def _subcontrib_to_marcxml(self, obj):
        self.xml_generator.openTag('record')
        self.output_generator.subContribToXMLMarc21(obj, out=self.xml_generator, overrideCache=True)
        self.xml_generator.closeTag('record')<commit_msg>Make XML generation more robust<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from flask_pluginengine import current_plugin

from MaKaC.accessControl import AccessWrapper, AdminList
from MaKaC.common.output import outputGenerator
from MaKaC.common.xmlGen import XMLGen

from indico_livesync import process_records, SimpleChange
from indico_livesync.util import make_compound_id, obj_deref, obj_ref


class MARCXMLGenerator:
    """Generates MARCXML based on Indico objects"""

    @classmethod
    def records_to_xml(cls, records):
        mg = MARCXMLGenerator()
        for ref, change in process_records(records).iteritems():
            mg.safe_add_object(ref, bool(change & SimpleChange.deleted))
        return mg.get_xml()

    @classmethod
    def objects_to_xml(cls, objs, change_type=SimpleChange.created):
        mg = MARCXMLGenerator()
        for obj in objs:
            mg.safe_add_object(obj_ref(obj), bool(change_type & SimpleChange.deleted))
        return mg.get_xml()

    def __init__(self):
        self.closed = False
        self.xml_generator = XMLGen()
        self.xml_generator.initXml()
        self.xml_generator.openTag('collection', [['xmlns', 'http://www.loc.gov/MARC21/slim']])
        # This is horrible. but refactoring all the code in the indico core would be just as bad.
        aw = AccessWrapper()
        aw.setUser(AdminList().getInstance().getList()[0])
        self.output_generator = outputGenerator(aw, self.xml_generator)

    def safe_add_object(self, ref, deleted=False):
        try:
            self.add_object(ref, deleted)
        except Exception:
            current_plugin.logger.exception('Could not process {}'.format(ref))

    def add_object(self, ref, deleted=False):
        if self.closed:
            raise RuntimeError('Cannot add object to closed xml generator')
        if deleted:
            xg = XMLGen(init=False)
            xg.openTag('record')
            xg.openTag('datafield', [['tag', '970'], ['ind1', ' '], ['ind2', ' ']])
            xg.writeTag('subfield', 'INDICO.{}'.format(make_compound_id(ref)), [['code', 'a']])
            xg.closeTag('datafield')
            xg.openTag('datafield', [['tag', '980'], ['ind1', ' '], ['ind2', ' ']])
            xg.writeTag('subfield', 'DELETED', [['code', 'c']])
            xg.closeTag('datafield')
            xg.closeTag('record')
            self.xml_generator += xg.xml
        elif ref['type'] in {'event', 'contribution', 'subcontribution'}:
            obj = obj_deref(ref)
            if obj is None:
                raise ValueError('Cannot add deleted object')
            elif not obj.getOwner():
                raise ValueError('Cannot add object without owner: {}'.format(obj))
            if ref['type'] == 'event':
                self.xml_generator.xml += self._event_to_marcxml(obj)
            elif ref['type'] == 'contribution':
                self.xml_generator.xml += self._contrib_to_marcxml(obj)
            elif ref['type'] == 'subcontribution':
                self.xml_generator.xml += self._subcontrib_to_marcxml(obj)
        else:
            raise ValueError('unknown object ref: {}'.format(ref['type']))
        return self.xml_generator.getXml()

    def get_xml(self):
        if not self.closed:
            self.xml_generator.closeTag('collection')
        return self.xml_generator.getXml()

    def _event_to_marcxml(self, obj):
        xg = XMLGen(init=False)
        xg.openTag('record')
        self.output_generator.confToXMLMarc21(obj, out=xg, overrideCache=True)
        xg.closeTag('record')
        return xg.xml

    def _contrib_to_marcxml(self, obj):
        xg = XMLGen(init=False)
        xg.openTag('record')
        self.output_generator.contribToXMLMarc21(obj, out=xg, overrideCache=True)
        xg.closeTag('record')
        return xg.xml

    def _subcontrib_to_marcxml(self, obj):
        xg = XMLGen(init=False)
        xg.openTag('record')
        self.output_generator.subContribToXMLMarc21(obj, out=xg, overrideCache=True)
        xg.closeTag('record')
        return xg.xml
{'fraction_non_alphanumeric': 0.11352835568697855, 'fraction_numerical': 0.006019643045728166, 'mean_word_length': 2.9474781158816175, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 74}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3813085', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2712, 'n_words': 710}