<commit_before># -*- coding: UTF-8
#   submission
#   **********
#
#   Implements a GlobaLeaks submission, then the operations performed
#   by an HTTP client in /submission URI

from twisted.internet.defer import inlineCallbacks

from globaleaks.settings import transact
from globaleaks.models import *
from globaleaks import security
from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, unauthenticated
from globaleaks.jobs.notification_sched import APSNotification
from globaleaks.jobs.delivery_sched import APSDelivery
from globaleaks.runner import GLAsynchronous
from globaleaks.rest import requests
from globaleaks.utils import log, utc_future_date, pretty_date_time, datetime_now, naturalize_fields
from globaleaks.third_party import rstr
from globaleaks.rest import errors


def wb_serialize_internaltip(internaltip):
    response = {
        'id' : unicode(internaltip.id),
        # compatibility! until client is not patched.
        'submission_gus' : unicode(internaltip.id),
        'context_gus': unicode(internaltip.context_id),
        'creation_date' : unicode(pretty_date_time(internaltip.creation_date)),
        'expiration_date' : unicode(pretty_date_time(internaltip.expiration_date)),
        'wb_fields' : dict(internaltip.wb_fields or {}),
        'download_limit' : int(internaltip.download_limit),
        'access_limit' : int(internaltip.access_limit),
        'mark' : unicode(internaltip.mark),
        'pertinence' : unicode(internaltip.pertinence_counter),
        'escalation_threshold' : unicode(internaltip.escalation_threshold),
        'files' : [],
        'receivers' : []
    }
    for receiver in internaltip.receivers:
        response['receivers'].append(receiver.id)

    for internalfile in internaltip.internalfiles:
        response['files'].append(internalfile.id)

    return response

@transact
def create_whistleblower_tip(store, submission_desc):
    """
    The plaintext receipt is returned only now, and then is
    stored hashed in the WBtip table
    """
    from Crypto import Random
    Random.atfork()

    assert submission_desc is not None and submission_desc.has_key('id')

    wbtip = WhistleblowerTip()

    context = store.find(Context, Context.id == submission_desc['context_gus']).one()

    return_value_receipt = unicode( rstr.xeger(context.receipt_regexp) )
    node = store.find(Node).one()
    wbtip.receipt_hash = security.hash_password(return_value_receipt, node.receipt_salt)

    wbtip.access_counter = 0
    wbtip.internaltip_id = submission_desc['id']
    store.add(wbtip)

    return return_value_receipt


# Remind: it's a store without @transaction because called by a @ŧransact
def import_receivers(store, submission, receiver_id_list, required=False):
    context = submission.context

    # As first we check if Context has some policies
    if not context.selectable_receiver:
        for receiver in context.receivers:
            # Add only the receiver not yet associated in Many-to-Many
            check = store.find(ReceiverInternalTip,
                ( ReceiverInternalTip.receiver_id == receiver.id,
                  ReceiverInternalTip.internaltip_id == submission.id) ).one()
            if not check:
                submission.receivers.add(receiver)

        store.commit()

        reloaded_submission = store.find(InternalTip, InternalTip.id == submission.id).one()
        log.debug("Context [%s] has a fixed receivers corpus #%d SID = %s" %
                (reloaded_submission.context.name[GLSetting.memory_copy.default_language],
                 reloaded_submission.receivers.count(), submission.id) )
        return

    # Clean the previous list of selected Receiver
    for prevrec in submission.receivers:
        try:
            submission.receivers.remove(prevrec)
        except Exception as excep:
            log.err("Unable to remove receiver from Tip, before new reassignment")
            raise excep

    store.commit()

    # without contexts policies, import WB requests and checks consistency
    sorted_receiver_id_list = list(set(receiver_id_list))
    for receiver_id in sorted_receiver_id_list:
        try:
            receiver = store.find(Receiver, Receiver.id == unicode(receiver_id)).one()
        except Exception as excep:
            log.err("Receiver requested (%s) can't be found: %s" %
                    (receiver_id, excep))
            raise errors.ReceiverGusNotFound

        if not context in receiver.contexts:
            raise errors.InvalidInputFormat("Forged receiver selection, you fuzzer! <:")

        try:
            submission.receivers.add(receiver)
        except Exception as excep:
            log.err("Receiver %s can't be assigned to the tip [%s]" % (receiver_id, excep) )
            continue

        log.debug("+receiver [%s] In tip (%s) #%d" %\
                (receiver.name, submission.id, submission.receivers.count() ) )

    if required and submission.receivers.count() == 0:
        log.err("Receivers required to be selected, not empty")
        raise errors.SubmissionFailFields("Needed almost one Receiver selected")


# Remind: it's a store without @transaction because called by a @ŧransact
def import_files(store, submission, files):
    """
    @param submission: the Storm obj
    @param files: the list of InternalFiles UUIDs
    @return:
        Look if all the files specified in the list exist
    """
    for file_id in files:
        try:
            ifile = store.find(InternalFile, InternalFile.id == unicode(file_id)).one()
        except Exception as excep:
            log.err("Storm error, not found %s file in import_files (%s)" %
                    (file_id, excep))
            raise errors.FileGusNotFound

        ifile.internaltip_id = submission.id

    # commit before return
    store.commit()


def import_fields(submission, wb_fields, configured_fields_list, strict_validation=False):
    """
    @param submission: the Storm object
    @param wb_fields: the received wb_fields
    @param configured_fields: the Context defined wb_fields
    @return: update the object of raise an Exception if a required field
        is missing, or if received field do not match the expected shape

    strict_validation = required the presence of 'required' wb_fields. Is not enforced
    if Submission would not be finalized yet.
    """
    required_keys = list()
    optional_keys  = list()

    assert isinstance(configured_fields_list, list), "context expected wb_fields"
    assert isinstance(wb_fields, dict), "receiver wb_fields"

    if not wb_fields and not strict_validation:
        return

    if strict_validation and not wb_fields:

        if not wb_fields:
            log.err("Missing submission in 'finalize' request")
            raise errors.SubmissionFailFields("Missing submission!")

    try:
        for single_field in configured_fields_list:
            if single_field['required'] == True:
                required_keys.append(single_field.get(u'key'))
            else:
                optional_keys.append(single_field.get(u'key'))
    except Exception, e:
        log.exception(e)
        raise errors.SubmissionFailFields("Malformed submission!")

    if strict_validation:

        log.debug("strict validation: %s (optional %s)" % (required_keys, optional_keys))

        for required in required_keys:

            if wb_fields.has_key(required) and len(wb_fields[required]) > 4:
            # the keys are always provided by GLClient! also if the content is empty.
            # then is not difficult check a test len(text) > $blah, but other checks are...
                continue

            log.err("Submission has a required field (%s) missing" % required)
            raise errors.SubmissionFailFields("Missing field '%s': Required" % required)

    if not wb_fields:
        return

    imported_fields = {}
    for key, value in wb_fields.iteritems():<commit_msg>fix against issue https://github.com/globaleaks/GlobaLeaks/issues/558<commit_after># -*- coding: UTF-8
#   submission
#   **********
#
#   Implements a GlobaLeaks submission, then the operations performed
#   by an HTTP client in /submission URI

from twisted.internet.defer import inlineCallbacks

from globaleaks.settings import transact
from globaleaks.models import *
from globaleaks import security
from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.authentication import transport_security_check, unauthenticated
from globaleaks.jobs.notification_sched import APSNotification
from globaleaks.jobs.delivery_sched import APSDelivery
from globaleaks.runner import GLAsynchronous
from globaleaks.rest import requests
from globaleaks.utils import log, utc_future_date, pretty_date_time, datetime_now, naturalize_fields
from globaleaks.third_party import rstr
from globaleaks.rest import errors


def wb_serialize_internaltip(internaltip):
    response = {
        'id' : unicode(internaltip.id),
        # compatibility! until client is not patched.
        'submission_gus' : unicode(internaltip.id),
        'context_gus': unicode(internaltip.context_id),
        'creation_date' : unicode(pretty_date_time(internaltip.creation_date)),
        'expiration_date' : unicode(pretty_date_time(internaltip.expiration_date)),
        'wb_fields' : dict(internaltip.wb_fields or {}),
        'download_limit' : int(internaltip.download_limit),
        'access_limit' : int(internaltip.access_limit),
        'mark' : unicode(internaltip.mark),
        'pertinence' : unicode(internaltip.pertinence_counter),
        'escalation_threshold' : unicode(internaltip.escalation_threshold),
        'files' : [],
        'receivers' : []
    }
    for receiver in internaltip.receivers:
        response['receivers'].append(receiver.id)

    for internalfile in internaltip.internalfiles:
        response['files'].append(internalfile.id)

    return response

@transact
def create_whistleblower_tip(store, submission_desc):
    """
    The plaintext receipt is returned only now, and then is
    stored hashed in the WBtip table
    """
    from Crypto import Random
    Random.atfork()

    assert submission_desc is not None and submission_desc.has_key('id')

    wbtip = WhistleblowerTip()

    context = store.find(Context, Context.id == submission_desc['context_gus']).one()

    return_value_receipt = unicode( rstr.xeger(context.receipt_regexp) )
    node = store.find(Node).one()
    wbtip.receipt_hash = security.hash_password(return_value_receipt, node.receipt_salt)

    wbtip.access_counter = 0
    wbtip.internaltip_id = submission_desc['id']
    store.add(wbtip)

    return return_value_receipt


# Remind: it's a store without @transaction because called by a @ŧransact
def import_receivers(store, submission, receiver_id_list, required=False):
    context = submission.context

    # As first we check if Context has some policies
    if not context.selectable_receiver:
        for receiver in context.receivers:
            # Add only the receiver not yet associated in Many-to-Many
            check = store.find(ReceiverInternalTip,
                ( ReceiverInternalTip.receiver_id == receiver.id,
                  ReceiverInternalTip.internaltip_id == submission.id) ).one()
            if not check:
                submission.receivers.add(receiver)

        store.commit()

        reloaded_submission = store.find(InternalTip, InternalTip.id == submission.id).one()
        log.debug("Context [%s] has a fixed receivers corpus #%d SID = %s" %
                (reloaded_submission.context.name[GLSetting.memory_copy.default_language],
                 reloaded_submission.receivers.count(), submission.id) )
        return

    # Clean the previous list of selected Receiver
    for prevrec in submission.receivers:
        try:
            submission.receivers.remove(prevrec)
        except Exception as excep:
            log.err("Unable to remove receiver from Tip, before new reassignment")
            raise excep

    store.commit()

    # without contexts policies, import WB requests and checks consistency
    sorted_receiver_id_list = list(set(receiver_id_list))
    for receiver_id in sorted_receiver_id_list:
        try:
            receiver = store.find(Receiver, Receiver.id == unicode(receiver_id)).one()
        except Exception as excep:
            log.err("Receiver requested (%s) can't be found: %s" %
                    (receiver_id, excep))
            raise errors.ReceiverGusNotFound

        if not context in receiver.contexts:
            raise errors.InvalidInputFormat("Forged receiver selection, you fuzzer! <:")

        try:
            submission.receivers.add(receiver)
        except Exception as excep:
            log.err("Receiver %s can't be assigned to the tip [%s]" % (receiver_id, excep) )
            continue

        log.debug("+receiver [%s] In tip (%s) #%d" %\
                (receiver.name, submission.id, submission.receivers.count() ) )

    if required and submission.receivers.count() == 0:
        log.err("Receivers required to be selected, not empty")
        raise errors.SubmissionFailFields("Needed almost one Receiver selected")


# Remind: it's a store without @transaction because called by a @ŧransact
def import_files(store, submission, files):
    """
    @param submission: the Storm obj
    @param files: the list of InternalFiles UUIDs
    @return:
        Look if all the files specified in the list exist
    """
    for file_id in files:
        try:
            ifile = store.find(InternalFile, InternalFile.id == unicode(file_id)).one()
        except Exception as excep:
            log.err("Storm error, not found %s file in import_files (%s)" %
                    (file_id, excep))
            raise errors.FileGusNotFound

        ifile.internaltip_id = submission.id

    # commit before return
    store.commit()


def import_fields(submission, wb_fields, configured_fields_list, strict_validation=False):
    """
    @param submission: the Storm object
    @param wb_fields: the received wb_fields
    @param configured_fields: the Context defined wb_fields
    @return: update the object of raise an Exception if a required field
        is missing, or if received field do not match the expected shape

    strict_validation = required the presence of 'required' wb_fields. Is not enforced
    if Submission would not be finalized yet.
    """
    required_keys = list()
    optional_keys  = list()

    assert isinstance(configured_fields_list, list), "context expected wb_fields"
    assert isinstance(wb_fields, dict), "receiver wb_fields"

    if not wb_fields and not strict_validation:
        return

    if strict_validation and not wb_fields:

        if not wb_fields:
            log.err("Missing submission in 'finalize' request")
            raise errors.SubmissionFailFields("Missing submission!")

    try:
        for single_field in configured_fields_list:
            if single_field['required'] == True:
                required_keys.append(single_field.get(u'key'))
            else:
                optional_keys.append(single_field.get(u'key'))
    except Exception, e:
        log.exception(e)
        raise errors.SubmissionFailFields("Malformed submission!")

    if strict_validation:

        log.debug("strict validation: %s (optional %s)" % (required_keys, optional_keys))

        for required in required_keys:

            if wb_fields.has_key(required) and len(wb_fields[required]) > 0:
            # the keys are always provided by GLClient! also if the content is empty.
            # then is not difficult check a test len(text) > $blah, but other checks are...
                continue

            log.err("Submission has a required field (%s) missing" % required)
            raise errors.SubmissionFailFields("Missing field '%s': Required" % required)

    if not wb_fields:
        return

    imported_fields = {}
    for key, value in wb_fields.iteritems():
{'fraction_non_alphanumeric': 0.08627156789197299, 'fraction_numerical': 0.0006876719179794949, 'mean_word_length': 3.0488483928119465, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798896', 'n_tokens_mistral': 4625, 'n_tokens_neox': 4432, 'n_words': 1250}