<commit_before># -*- coding: UTF-8
#
# submission
# **********
#
# Implements a GlobaLeaks submission, then the operations performed
#   by an HTTP client in /submission URI

import copy

import json

from storm.expr import And, In

from twisted.internet import defer

from globaleaks import models
from globaleaks.orm import transact
from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.admin.context import db_get_context_steps
from globaleaks.handlers.authentication import transport_security_check, unauthenticated, get_tor2web_header
from globaleaks.utils.token import TokenList
from globaleaks.rest import errors, requests
from globaleaks.security import hash_password, sha256, generateRandomReceipt
from globaleaks.settings import GLSettings
from globaleaks.utils.structures import Rosetta, get_localized_values
from globaleaks.utils.utility import log, utc_future_date, datetime_now, datetime_to_ISO8601, ISO8601_to_datetime


def get_submission_sequence_number(itip):
    return "%s-%d" % (itip.creation_date.strftime("%Y%m%d"), itip.progressive)


def db_assign_submission_progressive(store):
    counter = store.find(models.Counter, models.Counter.key == u'submission_sequence').one()
    if not counter:
        counter = models.Counter({'key': u'submission_sequence'})
        store.add(counter)
    else:
        now = datetime_now()
        update = counter.update_date
        if ((now > counter.update_date) and (not((now.year == update.year) and \
                                                 (now.month == update.month) and \
                                                 (now.day == update.day)))):
            counter.counter = 1
        else:
            counter.counter += 1

        counter.update_date = now

    return counter.counter


def _db_get_archived_field_recursively(field, language):
    for key, value in field.get('attrs', {}).iteritems():
        if key not in field['attrs']: continue
        if 'type' not in field['attrs'][key]: continue

        if field['attrs'][key]['type'] == u'localized':
            if language in field['attrs'][key].get('value', []):
                field['attrs'][key]['value'] = field['attrs'][key]['value'][language]
            else:
                field['attrs'][key]['value'] = ""

    for o in field.get('options', []):
        get_localized_values(o, o, models.FieldOption.localized_keys, language)

    for c in field.get('children', []):
        _db_get_archived_field_recursively(c, language)

    return get_localized_values(field, field, models.Field.localized_keys, language)


def _db_get_archived_questionnaire_schema(store, hash, type, language):
    aqs = store.find(models.ArchivedSchema,
                     models.ArchivedSchema.hash == hash,
                     models.ArchivedSchema.type == type).one()

    if not aqs:
        log.err("Unable to find questionnaire schema with hash %s" % hash)
        questionnaire = []
    else:
        questionnaire = copy.deepcopy(aqs.schema)

    for step in questionnaire:
        for field in step['children']:
            _db_get_archived_field_recursively(field, language)

        get_localized_values(step, step, models.Step.localized_keys, language)

    return questionnaire


def db_get_archived_questionnaire_schema(store, hash, language):
    return _db_get_archived_questionnaire_schema(store, hash, u'questionnaire', language)


def db_get_archived_preview_schema(store, hash, language):
    return _db_get_archived_questionnaire_schema(store, hash, u'preview', language)


def db_serialize_questionnaire_answers_recursively(answers):
    ret = {}

    for answer in answers:
        if answer.is_leaf:
            ret[answer.key] = answer.value
        else:<commit_msg>Fix localization of tip preview<commit_after># -*- coding: UTF-8
#
# submission
# **********
#
# Implements a GlobaLeaks submission, then the operations performed
#   by an HTTP client in /submission URI

import copy

import json

from storm.expr import And, In

from twisted.internet import defer

from globaleaks import models
from globaleaks.orm import transact
from globaleaks.handlers.base import BaseHandler
from globaleaks.handlers.admin.context import db_get_context_steps
from globaleaks.handlers.authentication import transport_security_check, unauthenticated, get_tor2web_header
from globaleaks.utils.token import TokenList
from globaleaks.rest import errors, requests
from globaleaks.security import hash_password, sha256, generateRandomReceipt
from globaleaks.settings import GLSettings
from globaleaks.utils.structures import Rosetta, get_localized_values
from globaleaks.utils.utility import log, utc_future_date, datetime_now, datetime_to_ISO8601, ISO8601_to_datetime


def get_submission_sequence_number(itip):
    return "%s-%d" % (itip.creation_date.strftime("%Y%m%d"), itip.progressive)


def db_assign_submission_progressive(store):
    counter = store.find(models.Counter, models.Counter.key == u'submission_sequence').one()
    if not counter:
        counter = models.Counter({'key': u'submission_sequence'})
        store.add(counter)
    else:
        now = datetime_now()
        update = counter.update_date
        if ((now > counter.update_date) and (not((now.year == update.year) and \
                                                 (now.month == update.month) and \
                                                 (now.day == update.day)))):
            counter.counter = 1
        else:
            counter.counter += 1

        counter.update_date = now

    return counter.counter


def _db_get_archived_field_recursively(field, language):
    for key, value in field.get('attrs', {}).iteritems():
        if key not in field['attrs']: continue
        if 'type' not in field['attrs'][key]: continue

        if field['attrs'][key]['type'] == u'localized':
            if language in field['attrs'][key].get('value', []):
                field['attrs'][key]['value'] = field['attrs'][key]['value'][language]
            else:
                field['attrs'][key]['value'] = ""

    for o in field.get('options', []):
        get_localized_values(o, o, models.FieldOption.localized_keys, language)

    for c in field.get('children', []):
        _db_get_archived_field_recursively(c, language)

    return get_localized_values(field, field, models.Field.localized_keys, language)


def _db_get_archived_questionnaire_schema(store, hash, type, language):
    aqs = store.find(models.ArchivedSchema,
                     models.ArchivedSchema.hash == hash,
                     models.ArchivedSchema.type == type).one()

    if not aqs:
        log.err("Unable to find questionnaire schema with hash %s" % hash)
        questionnaire = []
    else:
        questionnaire = copy.deepcopy(aqs.schema)

    if type == 'questionnaire':
        for step in questionnaire:
            for field in step['children']:
                _db_get_archived_field_recursively(field, language)

            get_localized_values(step, step, models.Step.localized_keys, language)

    elif type == 'preview':
        for field in questionnaire:
            _db_get_archived_field_recursively(field, language)

    return questionnaire


def db_get_archived_questionnaire_schema(store, hash, language):
    return _db_get_archived_questionnaire_schema(store, hash, u'questionnaire', language)


def db_get_archived_preview_schema(store, hash, language):
    return _db_get_archived_questionnaire_schema(store, hash, u'preview', language)


def db_serialize_questionnaire_answers_recursively(answers):
    ret = {}

    for answer in answers:
        if answer.is_leaf:
            ret[answer.key] = answer.value
        else:
{'fraction_non_alphanumeric': 0.1025107324053597, 'fraction_numerical': 0.0039026928580720697, 'mean_word_length': 3.2806236080178173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4727201', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2177, 'n_words': 525}