<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import traceback
from datetime import timedelta
from operator import attrgetter

import click
from pytz import utc
from sqlalchemy.orm import joinedload

from indico.core.db import db
from indico.modules.events import Event
from indico.modules.events.abstracts.models.abstracts import Abstract, AbstractState
from indico.modules.events.abstracts.models.email_templates import AbstractEmailTemplate
from indico.modules.events.abstracts.settings import (abstracts_settings, boa_settings,
                                                      BOACorrespondingAuthorType, BOASortField)
from indico.modules.events.models.events import EventType
from indico.modules.events.tracks.models.tracks import Track
from indico.modules.users import User
from indico.modules.users.legacy import AvatarUserWrapper
from indico.util.console import verbose_iterator, cformat
from indico.util.string import sanitize_email
from indico.util.struct.iterables import committing_iterator

from indico_zodbimport import Importer, convert_to_unicode


def strict_sanitize_email(email, fallback=None):
    return sanitize_email(convert_to_unicode(email).lower(), require_valid=True) or fallback


class AbstractMigration(object):
    CONDITION_MAP = {'NotifTplCondAccepted': AbstractState.accepted,
                     'NotifTplCondRejected': AbstractState.rejected,
                     'NotifTplCondMerged': AbstractState.merged}

    def __init__(self, importer, conf, event):
        self.importer = importer
        self.conf = conf
        self.event = event
        self.amgr = conf.abstractMgr
        self.track_map = {}
        self.legacy_warnings_shown = set()

    def __repr__(self):
        return '<AbstractMigration({})>'.format(self.event)

    def run(self):
        self.importer.print_success(cformat('%{blue!}{}').format(self.event), event_id=self.event.id)
        self._migrate_tracks()
        self._migrate_boa_settings()
        self._migrate_settings()
        self._migrate_email_templates()
        # TODO...

    def _user_from_legacy(self, legacy_user):
        if isinstance(legacy_user, AvatarUserWrapper):
            user = legacy_user.user
            email = convert_to_unicode(legacy_user.__dict__.get('email', '')).lower() or None
        elif legacy_user.__class__.__name__ == 'Avatar':
            user = AvatarUserWrapper(legacy_user.id).user
            email = convert_to_unicode(legacy_user.email).lower()
        else:
            self.importer.print_error(cformat('%{red!}Invalid legacy user: {}').format(legacy_user),
                                      event_id=self.event.id)
            return None
        if user is None:
            user = self.importer.all_users_by_email.get(email) if email else None
            if user is not None:
                msg = 'Using {} for {} (matched via {})'.format(user, legacy_user, email)
            else:
                msg = cformat('%{yellow}Invalid legacy user: {}').format(legacy_user)
            self.importer.print_warning(msg, event_id=self.event.id, always=(msg not in self.legacy_warnings_shown))
            self.legacy_warnings_shown.add(msg)
        return user

    def _event_to_utc(self, dt):
        return self.event.tzinfo.localize(dt).astimezone(utc)

    def _migrate_tracks(self):
        for pos, old_track in enumerate(self.conf.program, 1):
            track = Track(title=convert_to_unicode(old_track.title),
                          description=convert_to_unicode(old_track.description),
                          code=convert_to_unicode(old_track._code),
                          position=pos,
                          abstract_reviewers=set())
            self.importer.print_info(cformat('%{white!}Track:%{reset} {}').format(track.title))
            for coordinator in old_track._coordinators:
                user = self._user_from_legacy(coordinator)
                if user is None:
                    continue
                self.importer.print_info(cformat('%{blue!}  Coordinator:%{reset} {}').format(user))
                track.abstract_reviewers.add(user)
                self.event.update_principal(user, add_roles={'abstract_reviewer'}, quiet=True)
            self.track_map[old_track] = track
            self.event.tracks.append(track)

    def _migrate_boa_settings(self):
        boa_config = self.conf._boa
        sort_field_map = {'number': 'id', 'none': 'id', 'name': 'abstract_title', 'sessionTitle': 'session_title',
                          'speakers': 'speaker', 'submitter': 'id'}
        try:
            sort_by = sort_field_map.get(boa_config._sortBy, boa_config._sortBy)
        except AttributeError:
            sort_by = 'id'
        corresponding_author = getattr(boa_config, '_correspondingAuthor', 'submitter')
        boa_settings.set_multi(self.event, {
            'extra_text': convert_to_unicode(boa_config._text),
            'sort_by': BOASortField[sort_by],
            'corresponding_author': BOACorrespondingAuthorType[corresponding_author],
            'show_abstract_ids': bool(getattr(boa_config, '_showIds', False))
        })

    def _migrate_settings(self):
        start_dt = self._event_to_utc(self.amgr._submissionStartDate)
        end_dt = self._event_to_utc(self.amgr._submissionEndDate)
        modification_end_dt = (self._event_to_utc(self.amgr._modifDeadline)
                               if getattr(self.amgr, '_modifDeadline', None)
                               else None)
        assert start_dt < end_dt
        if modification_end_dt and modification_end_dt - end_dt < timedelta(minutes=1):
            if modification_end_dt != end_dt:
                self.importer.print_warning('Ignoring mod deadline ({} > {})'.format(end_dt, modification_end_dt),
                                            event_id=self.event.id)
            modification_end_dt = None
        abstracts_settings.set_multi(self.event, {
            'start_dt': start_dt,
            'end_dt': end_dt,
            'modification_end_dt': modification_end_dt,
            'announcement': convert_to_unicode(self.amgr._announcement),
            'allow_multiple_tracks': bool(getattr(self.amgr, '_multipleTracks', True)),
            'tracks_required': bool(getattr(self.amgr, '_tracksMandatory', False)),
            'allow_attachments': bool(getattr(self.amgr, '_attachFiles', False)),
            'allow_speakers': bool(getattr(self.amgr, '_showSelectAsSpeaker', True)),
            'speakers_required': bool(getattr(self.amgr, '_selectSpeakerMandatory', True)),
            'authorized_submitters': set(filter(None, map(self._user_from_legacy, self.amgr._authorizedSubmitter)))
        })

    def _convert_email_template(self, tpl):
        placeholders = {'abstract_URL': 'abstract_url',
                        'abstract_id': 'abstract_id',
                        'abstract_review_comments': 'judgment_comment',
                        'abstract_session': 'abstract_session',
                        'abstract_title': 'abstract_title',<commit_msg>Migrate review settings & questions<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import traceback
from datetime import timedelta
from operator import attrgetter

import click
from pytz import utc
from sqlalchemy.orm import joinedload

from indico.core.db import db
from indico.modules.events import Event
from indico.modules.events.abstracts.models.abstracts import Abstract, AbstractState
from indico.modules.events.abstracts.models.email_templates import AbstractEmailTemplate
from indico.modules.events.abstracts.models.review_questions import AbstractReviewQuestion
from indico.modules.events.abstracts.settings import (abstracts_settings, boa_settings, abstracts_reviewing_settings,
                                                      BOACorrespondingAuthorType, BOASortField)
from indico.modules.events.models.events import EventType
from indico.modules.events.tracks.models.tracks import Track
from indico.modules.users import User
from indico.modules.users.legacy import AvatarUserWrapper
from indico.util.console import verbose_iterator, cformat
from indico.util.string import sanitize_email
from indico.util.struct.iterables import committing_iterator

from indico_zodbimport import Importer, convert_to_unicode


def strict_sanitize_email(email, fallback=None):
    return sanitize_email(convert_to_unicode(email).lower(), require_valid=True) or fallback


class AbstractMigration(object):
    CONDITION_MAP = {'NotifTplCondAccepted': AbstractState.accepted,
                     'NotifTplCondRejected': AbstractState.rejected,
                     'NotifTplCondMerged': AbstractState.merged}

    def __init__(self, importer, conf, event):
        self.importer = importer
        self.conf = conf
        self.event = event
        self.amgr = conf.abstractMgr
        self.track_map = {}
        self.question_map = {}
        self.legacy_warnings_shown = set()

    def __repr__(self):
        return '<AbstractMigration({})>'.format(self.event)

    def run(self):
        self.importer.print_success(cformat('%{blue!}{}').format(self.event), event_id=self.event.id)
        self._migrate_tracks()
        self._migrate_boa_settings()
        self._migrate_settings()
        self._migrate_review_settings()
        self._migrate_email_templates()
        # TODO...

    def _user_from_legacy(self, legacy_user):
        if isinstance(legacy_user, AvatarUserWrapper):
            user = legacy_user.user
            email = convert_to_unicode(legacy_user.__dict__.get('email', '')).lower() or None
        elif legacy_user.__class__.__name__ == 'Avatar':
            user = AvatarUserWrapper(legacy_user.id).user
            email = convert_to_unicode(legacy_user.email).lower()
        else:
            self.importer.print_error(cformat('%{red!}Invalid legacy user: {}').format(legacy_user),
                                      event_id=self.event.id)
            return None
        if user is None:
            user = self.importer.all_users_by_email.get(email) if email else None
            if user is not None:
                msg = 'Using {} for {} (matched via {})'.format(user, legacy_user, email)
            else:
                msg = cformat('%{yellow}Invalid legacy user: {}').format(legacy_user)
            self.importer.print_warning(msg, event_id=self.event.id, always=(msg not in self.legacy_warnings_shown))
            self.legacy_warnings_shown.add(msg)
        return user

    def _event_to_utc(self, dt):
        return self.event.tzinfo.localize(dt).astimezone(utc)

    def _migrate_tracks(self):
        for pos, old_track in enumerate(self.conf.program, 1):
            track = Track(title=convert_to_unicode(old_track.title),
                          description=convert_to_unicode(old_track.description),
                          code=convert_to_unicode(old_track._code),
                          position=pos,
                          abstract_reviewers=set())
            self.importer.print_info(cformat('%{white!}Track:%{reset} {}').format(track.title))
            for coordinator in old_track._coordinators:
                user = self._user_from_legacy(coordinator)
                if user is None:
                    continue
                self.importer.print_info(cformat('%{blue!}  Coordinator:%{reset} {}').format(user))
                track.abstract_reviewers.add(user)
                self.event.update_principal(user, add_roles={'abstract_reviewer'}, quiet=True)
            self.track_map[old_track] = track
            self.event.tracks.append(track)

    def _migrate_boa_settings(self):
        boa_config = self.conf._boa
        sort_field_map = {'number': 'id', 'none': 'id', 'name': 'abstract_title', 'sessionTitle': 'session_title',
                          'speakers': 'speaker', 'submitter': 'id'}
        try:
            sort_by = sort_field_map.get(boa_config._sortBy, boa_config._sortBy)
        except AttributeError:
            sort_by = 'id'
        corresponding_author = getattr(boa_config, '_correspondingAuthor', 'submitter')
        boa_settings.set_multi(self.event, {
            'extra_text': convert_to_unicode(boa_config._text),
            'sort_by': BOASortField[sort_by],
            'corresponding_author': BOACorrespondingAuthorType[corresponding_author],
            'show_abstract_ids': bool(getattr(boa_config, '_showIds', False))
        })

    def _migrate_settings(self):
        start_dt = self._event_to_utc(self.amgr._submissionStartDate)
        end_dt = self._event_to_utc(self.amgr._submissionEndDate)
        modification_end_dt = (self._event_to_utc(self.amgr._modifDeadline)
                               if getattr(self.amgr, '_modifDeadline', None)
                               else None)
        assert start_dt < end_dt
        if modification_end_dt and modification_end_dt - end_dt < timedelta(minutes=1):
            if modification_end_dt != end_dt:
                self.importer.print_warning('Ignoring mod deadline ({} > {})'.format(end_dt, modification_end_dt),
                                            event_id=self.event.id)
            modification_end_dt = None
        abstracts_settings.set_multi(self.event, {
            'start_dt': start_dt,
            'end_dt': end_dt,
            'modification_end_dt': modification_end_dt,
            'announcement': convert_to_unicode(self.amgr._announcement),
            'allow_multiple_tracks': bool(getattr(self.amgr, '_multipleTracks', True)),
            'tracks_required': bool(getattr(self.amgr, '_tracksMandatory', False)),
            'allow_attachments': bool(getattr(self.amgr, '_attachFiles', False)),
            'allow_speakers': bool(getattr(self.amgr, '_showSelectAsSpeaker', True)),
            'speakers_required': bool(getattr(self.amgr, '_selectSpeakerMandatory', True)),
            'authorized_submitters': set(filter(None, map(self._user_from_legacy, self.amgr._authorizedSubmitter)))
        })

    def _migrate_review_settings(self):
        try:
            old_settings = self.conf._confAbstractReview
        except AttributeError:
            return
        abstracts_reviewing_settings.set_multi(self.event, {
            'num_answers': int(old_settings._numberOfAnswers),
            'scale_lower': int(old_settings._scaleLower),
            'scale_upper': int(old_settings._scaleHigher),
            'reviewers_final_judgement': bool(getattr(old_settings, '_canReviewerAccept', False))
        })
        for pos, old_question in enumerate(old_settings._reviewingQuestions, 1):
            question = AbstractReviewQuestion(position=pos, text=convert_to_unicode(old_question._text))
            self.question_map[old_question] = question
            self.event.abstract_review_questions.append(question)

    def _convert_email_template(self, tpl):
        placeholders = {'abstract_URL': 'abstract_url',
                        'abstract_id': 'abstract_id',
                        'abstract_review_comments': 'judgment_comment',
                        'abstract_session': 'abstract_session',
                        'abstract_title': 'abstract_title',
{'fraction_non_alphanumeric': 0.09644578313253012, 'fraction_numerical': 0.0013855421686746988, 'mean_word_length': 2.806695711992662, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4616802', 'n_tokens_mistral': 4789, 'n_tokens_neox': 4646, 'n_words': 980}