<commit_before># -*- coding: UTF-8
#
# rtip
# ****
#
# Contains all the logic for handling tip related operations, for the
# receiver side. These classes are executed in the /rtip/* URI PATH

import os

from twisted.internet.defer import inlineCallbacks

from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, authenticated
from globaleaks.handlers.submission import db_get_archived_questionnaire_schema, \
    db_serialize_questionnaire_answers
from globaleaks.rest import requests
from globaleaks.utils.utility import log, utc_future_date, datetime_now, \
    datetime_to_ISO8601, datetime_to_pretty_str

from globaleaks.utils.structures import Rosetta
from globaleaks.settings import transact, transact_ro, GLSettings
from globaleaks.models import Node, Notification, Comment, Message, \
    ReceiverFile, ReceiverTip, EventLogs,  InternalTip, ArchivedSchema, \
    SecureFileDelete
from globaleaks.rest import errors
from globaleaks.security import overwrite_and_remove


def receiver_serialize_tip(store, internaltip, language):
    ret_dict = {
        'id': internaltip.id,
        'context_id': internaltip.context.id,
        'show_receivers': internaltip.context.show_receivers,
        'creation_date': datetime_to_ISO8601(internaltip.creation_date),
        'expiration_date': datetime_to_ISO8601(internaltip.expiration_date),
        'questionnaire': db_get_archived_questionnaire_schema(store, internaltip.questionnaire_hash, language),
        'answers': db_serialize_questionnaire_answers(store, internaltip),
        'tor2web': internaltip.tor2web,
        'timetolive': internaltip.context.tip_timetolive,
        'enable_comments': internaltip.enable_comments,
        'enable_messages': internaltip.enable_messages,
        'enable_two_way_communication': internaltip.enable_two_way_communication,
        'enable_attachments': internaltip.enable_attachments
    }

    # context_name and context_description are localized fields
    mo = Rosetta(internaltip.context.localized_strings)
    mo.acquire_storm_object(internaltip.context)
    for attr in ['name', 'description']:
        key = "context_%s" % attr
        ret_dict[key] = mo.dump_localized_key(attr, language)

    return ret_dict


def receiver_serialize_file(internalfile, receiverfile, receivertip_id):
    """
    ReceiverFile is the mixing between the metadata present in InternalFile
    and the Receiver-dependent, and for the client sake receivertip_id is
    required to create the download link
    """
    if receiverfile.status != 'unavailable':

        ret_dict = {
            'id': receiverfile.id,
            'ifile_id': internalfile.id,
            'status': receiverfile.status,
            'href': "/rtip/" + receivertip_id + "/download/" + receiverfile.id,
            # if the ReceiverFile has encrypted status, we append ".pgp" to the filename, to avoid mistake on Receiver side.
            'name': ("%s.pgp" % internalfile.name) if receiverfile.status == u'encrypted' else internalfile.name,
            'content_type': internalfile.content_type,
            'creation_date': datetime_to_ISO8601(internalfile.creation_date),
            'size': receiverfile.size,
            'downloads': receiverfile.downloads
        }

    else:  # == 'unavailable' in this case internal file metadata is returned.

        ret_dict = {
            'id': receiverfile.id,
            'ifile_id': internalfile.id,
            'status': 'unavailable',
            'href': "",
            'name': internalfile.name,  # original filename
            'content_type': internalfile.content_type,  # original content size
            'creation_date': datetime_to_ISO8601(internalfile.creation_date),  # original creation_date
            'size': int(internalfile.size),  # original filesize
            'downloads': unicode(receiverfile.downloads)  # this counter is always valid
        }

    return ret_dict


def db_get_files_receiver(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    receiver_files = store.find(ReceiverFile,
                                (ReceiverFile.internaltip_id == rtip.internaltip_id,
                                 ReceiverFile.receiver_id == rtip.receiver_id))

    files_list = []
    for receiverfile in receiver_files:
        internalfile = receiverfile.internalfile
        files_list.append(receiver_serialize_file(internalfile, receiverfile, tip_id))

    return files_list


def db_get_tip_receiver(store, user_id, tip_id, language):
    rtip = db_access_tip(store, user_id, tip_id)

    notif = store.find(Notification).one()

    if not notif.send_email_for_every_event:
        # If Receiver is accessing this Tip, Events related can be removed before
        # Notification is sent. This is a Twitter/Facebook -like behavior.
        store.find(EventLogs, EventLogs.receivertip_id == tip_id).remove()
        # Note - before the check was:
        # store.find(EventLogs, And(EventLogs.receivertip_id == tip_id,
        #                          EventLogs.mail_sent == True)).remove()

    tip_desc = receiver_serialize_tip(store, rtip.internaltip, language)

    # are added here because part of ReceiverTip, not InternalTip
    tip_desc['access_counter'] = rtip.access_counter
    tip_desc['id'] = rtip.id
    tip_desc['receiver_id'] = user_id
    tip_desc['label'] = rtip.label

    return tip_desc


def db_increment_receiver_access_count(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    rtip.access_counter += 1
    rtip.last_access = datetime_now()

    log.debug("Tip %s access granted to user %s (%d)" %
              (rtip.id, rtip.receiver.user.name, rtip.access_counter))

    return rtip.access_counter


def db_access_tip(store, user_id, tip_id):
    rtip = store.find(ReceiverTip, ReceiverTip.id == unicode(tip_id),
                      ReceiverTip.receiver_id == user_id).one()

    if not rtip:
        raise errors.TipIdNotFound

    return rtip


def db_delete_itip(store, itip):
    for ifile in itip.internalfiles:
        abspath = os.path.join(GLSettings.submission_path, ifile.file_path)

        if os.path.isfile(abspath):
            log.debug("Marking internalfile %s for secure deletion" % abspath)
            secure_file_delete = SecureFileDelete()
            secure_file_delete.filepath = abspath
            store.add(secure_file_delete)

        rfiles = store.find(ReceiverFile, ReceiverFile.internalfile_id == ifile.id)
        for rfile in rfiles:
            # The following code must be bypassed if rfile.file_path == ifile.filepath,
            # this mean that is referenced the plaintext file instead having E2E.
            if rfile.file_path == ifile.file_path:
                continue

            abspath = os.path.join(GLSettings.submission_path, rfile.file_path)

            if os.path.isfile(abspath):
                log.debug("Marking receiverfile %s for secure deletion" % abspath)
                secure_file_delete = SecureFileDelete()
                secure_file_delete.filepath = abspath
                store.add(secure_file_delete)

        log.debug("Removing InternalTip %s" % itip.id)

    store.remove(itip)

    if store.find(InternalTip, InternalTip.questionnaire_hash == itip.questionnaire_hash).count() == 0:
        store.find(ArchivedSchema, ArchivedSchema.hash == itip.questionnaire_hash).remove()


def db_delete_rtip(store, rtip):
    return db_delete_itip(store, rtip.internaltip)


def db_postpone_expiration_date(rtip):
    rtip.internaltip.expiration_date = \
        utc_future_date(seconds=rtip.internaltip.context.tip_timetolive)


@transact
def delete_rtip(store, user_id, tip_id):
    """
    Delete internalTip is possible only to Receiver with
    the dedicated property.
    """
    rtip = db_access_tip(store, user_id, tip_id)

    node = store.find(Node).one()

    if not (node.can_delete_submission or
                rtip.receiver.can_delete_submission):
        raise errors.ForbiddenOperation

    db_delete_rtip(store, rtip)


@transact
def postpone_expiration_date(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    node = store.find(Node).one()

    if not (node.can_postpone_expiration or
                rtip.receiver.can_postpone_expiration):

        raise errors.ExtendTipLifeNotEnabled

    log.debug("Postpone check: Node %s, Receiver %s" % (
       "True" if node.can_postpone_expiration else "False",
       "True" if rtip.receiver.can_postpone_expiration else "False"
    ))

    db_postpone_expiration_date(rtip)

    log.debug(" [%s] in %s has postponed expiration time to %s" % (
        rtip.receiver.user.name,
        datetime_to_pretty_str(datetime_now()),
        datetime_to_pretty_str(rtip.internaltip.expiration_date)))


@transact
def assign_rtip_label(store, user_id, tip_id, label_content):
    rtip = db_access_tip(store, user_id, tip_id)<commit_msg>Optimize postpone/delete queries by removing need for fetching Node table<commit_after># -*- coding: UTF-8
#
# rtip
# ****
#
# Contains all the logic for handling tip related operations, for the
# receiver side. These classes are executed in the /rtip/* URI PATH

import os

from twisted.internet.defer import inlineCallbacks

from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, authenticated
from globaleaks.handlers.submission import db_get_archived_questionnaire_schema, \
    db_serialize_questionnaire_answers
from globaleaks.rest import requests
from globaleaks.utils.utility import log, utc_future_date, datetime_now, \
    datetime_to_ISO8601, datetime_to_pretty_str

from globaleaks.utils.structures import Rosetta
from globaleaks.settings import transact, transact_ro, GLSettings
from globaleaks.models import Node, Notification, Comment, Message, \
    ReceiverFile, ReceiverTip, EventLogs,  InternalTip, ArchivedSchema, \
    SecureFileDelete
from globaleaks.rest import errors
from globaleaks.security import overwrite_and_remove


def receiver_serialize_tip(store, internaltip, language):
    ret_dict = {
        'id': internaltip.id,
        'context_id': internaltip.context.id,
        'show_receivers': internaltip.context.show_receivers,
        'creation_date': datetime_to_ISO8601(internaltip.creation_date),
        'expiration_date': datetime_to_ISO8601(internaltip.expiration_date),
        'questionnaire': db_get_archived_questionnaire_schema(store, internaltip.questionnaire_hash, language),
        'answers': db_serialize_questionnaire_answers(store, internaltip),
        'tor2web': internaltip.tor2web,
        'timetolive': internaltip.context.tip_timetolive,
        'enable_comments': internaltip.enable_comments,
        'enable_messages': internaltip.enable_messages,
        'enable_two_way_communication': internaltip.enable_two_way_communication,
        'enable_attachments': internaltip.enable_attachments
    }

    # context_name and context_description are localized fields
    mo = Rosetta(internaltip.context.localized_strings)
    mo.acquire_storm_object(internaltip.context)
    for attr in ['name', 'description']:
        key = "context_%s" % attr
        ret_dict[key] = mo.dump_localized_key(attr, language)

    return ret_dict


def receiver_serialize_file(internalfile, receiverfile, receivertip_id):
    """
    ReceiverFile is the mixing between the metadata present in InternalFile
    and the Receiver-dependent, and for the client sake receivertip_id is
    required to create the download link
    """
    if receiverfile.status != 'unavailable':

        ret_dict = {
            'id': receiverfile.id,
            'ifile_id': internalfile.id,
            'status': receiverfile.status,
            'href': "/rtip/" + receivertip_id + "/download/" + receiverfile.id,
            # if the ReceiverFile has encrypted status, we append ".pgp" to the filename, to avoid mistake on Receiver side.
            'name': ("%s.pgp" % internalfile.name) if receiverfile.status == u'encrypted' else internalfile.name,
            'content_type': internalfile.content_type,
            'creation_date': datetime_to_ISO8601(internalfile.creation_date),
            'size': receiverfile.size,
            'downloads': receiverfile.downloads
        }

    else:  # == 'unavailable' in this case internal file metadata is returned.

        ret_dict = {
            'id': receiverfile.id,
            'ifile_id': internalfile.id,
            'status': 'unavailable',
            'href': "",
            'name': internalfile.name,  # original filename
            'content_type': internalfile.content_type,  # original content size
            'creation_date': datetime_to_ISO8601(internalfile.creation_date),  # original creation_date
            'size': int(internalfile.size),  # original filesize
            'downloads': unicode(receiverfile.downloads)  # this counter is always valid
        }

    return ret_dict


def db_get_files_receiver(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    receiver_files = store.find(ReceiverFile,
                                (ReceiverFile.internaltip_id == rtip.internaltip_id,
                                 ReceiverFile.receiver_id == rtip.receiver_id))

    files_list = []
    for receiverfile in receiver_files:
        internalfile = receiverfile.internalfile
        files_list.append(receiver_serialize_file(internalfile, receiverfile, tip_id))

    return files_list


def db_get_tip_receiver(store, user_id, tip_id, language):
    rtip = db_access_tip(store, user_id, tip_id)

    notif = store.find(Notification).one()

    if not notif.send_email_for_every_event:
        # If Receiver is accessing this Tip, Events related can be removed before
        # Notification is sent. This is a Twitter/Facebook -like behavior.
        store.find(EventLogs, EventLogs.receivertip_id == tip_id).remove()
        # Note - before the check was:
        # store.find(EventLogs, And(EventLogs.receivertip_id == tip_id,
        #                          EventLogs.mail_sent == True)).remove()

    tip_desc = receiver_serialize_tip(store, rtip.internaltip, language)

    # are added here because part of ReceiverTip, not InternalTip
    tip_desc['access_counter'] = rtip.access_counter
    tip_desc['id'] = rtip.id
    tip_desc['receiver_id'] = user_id
    tip_desc['label'] = rtip.label

    return tip_desc


def db_increment_receiver_access_count(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    rtip.access_counter += 1
    rtip.last_access = datetime_now()

    log.debug("Tip %s access granted to user %s (%d)" %
              (rtip.id, rtip.receiver.user.name, rtip.access_counter))

    return rtip.access_counter


def db_access_tip(store, user_id, tip_id):
    rtip = store.find(ReceiverTip, ReceiverTip.id == unicode(tip_id),
                      ReceiverTip.receiver_id == user_id).one()

    if not rtip:
        raise errors.TipIdNotFound

    return rtip


def db_delete_itip(store, itip):
    for ifile in itip.internalfiles:
        abspath = os.path.join(GLSettings.submission_path, ifile.file_path)

        if os.path.isfile(abspath):
            log.debug("Marking internalfile %s for secure deletion" % abspath)
            secure_file_delete = SecureFileDelete()
            secure_file_delete.filepath = abspath
            store.add(secure_file_delete)

        rfiles = store.find(ReceiverFile, ReceiverFile.internalfile_id == ifile.id)
        for rfile in rfiles:
            # The following code must be bypassed if rfile.file_path == ifile.filepath,
            # this mean that is referenced the plaintext file instead having E2E.
            if rfile.file_path == ifile.file_path:
                continue

            abspath = os.path.join(GLSettings.submission_path, rfile.file_path)

            if os.path.isfile(abspath):
                log.debug("Marking receiverfile %s for secure deletion" % abspath)
                secure_file_delete = SecureFileDelete()
                secure_file_delete.filepath = abspath
                store.add(secure_file_delete)

        log.debug("Removing InternalTip %s" % itip.id)

    store.remove(itip)

    if store.find(InternalTip, InternalTip.questionnaire_hash == itip.questionnaire_hash).count() == 0:
        store.find(ArchivedSchema, ArchivedSchema.hash == itip.questionnaire_hash).remove()


def db_delete_rtip(store, rtip):
    return db_delete_itip(store, rtip.internaltip)


def db_postpone_expiration_date(rtip):
    rtip.internaltip.expiration_date = \
        utc_future_date(seconds=rtip.internaltip.context.tip_timetolive)


@transact
def delete_rtip(store, user_id, tip_id):
    """
    Delete internalTip is possible only to Receiver with
    the dedicated property.
    """
    rtip = db_access_tip(store, user_id, tip_id)

    if not (GLSettings.memory_copy.can_delete_submission or
                rtip.receiver.can_delete_submission):
        raise errors.ForbiddenOperation

    db_delete_rtip(store, rtip)


@transact
def postpone_expiration_date(store, user_id, tip_id):
    rtip = db_access_tip(store, user_id, tip_id)

    if not (GLSettings.memory_copy.can_postpone_expiration or
                rtip.receiver.can_postpone_expiration):

        raise errors.ExtendTipLifeNotEnabled

    log.debug("Postpone check: Node %s, Receiver %s" % (
       "True" if GLSettings.memory_copy.can_postpone_expiration else "False",
       "True" if rtip.receiver.can_postpone_expiration else "False"
    ))

    db_postpone_expiration_date(rtip)

    log.debug(" [%s] in %s has postponed expiration time to %s" % (
        rtip.receiver.user.name,
        datetime_to_pretty_str(datetime_now()),
        datetime_to_pretty_str(rtip.internaltip.expiration_date)))


@transact
def assign_rtip_label(store, user_id, tip_id, label_content):
    rtip = db_access_tip(store, user_id, tip_id)
{'fraction_non_alphanumeric': 0.08839902026274772, 'fraction_numerical': 0.0028946782453796484, 'mean_word_length': 3.263170384432843, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1347314', 'n_tokens_mistral': 5632, 'n_tokens_neox': 5439, 'n_words': 1188}