<commit_before>import itertools
from datetime import datetime

from flask import Blueprint, current_app, jsonify, request, url_for
from notifications_utils.letter_timings import (
    letter_can_be_cancelled,
    too_late_to_cancel_letter,
)
from notifications_utils.timezones import convert_utc_to_bst
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound

from app.aws import s3
from app.config import QueueNames
from app.dao import fact_notification_status_dao, notifications_dao
from app.dao.annual_billing_dao import set_default_free_allowance_for_service
from app.dao.api_key_dao import (
    expire_api_key,
    get_model_api_keys,
    get_unsigned_secret,
    save_model_api_key,
)
from app.dao.broadcast_service_dao import set_broadcast_service_type
from app.dao.dao_utils import dao_rollback, transaction
from app.dao.date_util import get_financial_year
from app.dao.fact_notification_status_dao import (
    fetch_monthly_template_usage_for_service,
    fetch_notification_status_for_service_by_month,
    fetch_notification_status_for_service_for_day,
    fetch_notification_status_for_service_for_today_and_7_previous_days,
    fetch_stats_for_all_services_by_date_range,
)
from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
from app.dao.organisation_dao import dao_get_organisation_by_service_id
from app.dao.returned_letters_dao import (
    fetch_most_recent_returned_letter,
    fetch_recent_returned_letter_count,
    fetch_returned_letter_summary,
    fetch_returned_letters,
)
from app.dao.service_contact_list_dao import (
    dao_archive_contact_list,
    dao_get_contact_list_by_id,
    dao_get_contact_lists,
    save_service_contact_list,
)
from app.dao.service_data_retention_dao import (
    fetch_service_data_retention,
    fetch_service_data_retention_by_id,
    fetch_service_data_retention_by_notification_type,
    insert_service_data_retention,
    update_service_data_retention,
)
from app.dao.service_email_reply_to_dao import (
    add_reply_to_email_address_for_service,
    archive_reply_to_email_address,
    dao_get_reply_to_by_id,
    dao_get_reply_to_by_service_id,
    update_reply_to_email_address,
)
from app.dao.service_guest_list_dao import (
    dao_add_and_commit_guest_list_contacts,
    dao_fetch_service_guest_list,
    dao_remove_service_guest_list,
)
from app.dao.service_letter_contact_dao import (
    add_letter_contact_for_service,
    archive_letter_contact,
    dao_get_letter_contact_by_id,
    dao_get_letter_contacts_by_service_id,
    update_letter_contact,
)
from app.dao.service_sms_sender_dao import (
    archive_sms_sender,
    dao_add_sms_sender_for_service,
    dao_get_service_sms_senders_by_id,
    dao_get_sms_senders_by_service_id,
    dao_update_service_sms_sender,
    update_existing_sms_sender_with_inbound_number,
)
from app.dao.services_dao import (
    dao_add_user_to_service,
    dao_archive_service,
    dao_create_service,
    dao_fetch_all_services,
    dao_fetch_all_services_by_user,
    dao_fetch_live_services_data,
    dao_fetch_service_by_id,
    dao_fetch_todays_stats_for_all_services,
    dao_fetch_todays_stats_for_service,
    dao_remove_user_from_service,
    dao_resume_service,
    dao_suspend_service,
    dao_update_service,
    get_services_by_partial_name,
)
from app.dao.templates_dao import dao_get_template_by_id
from app.dao.users_dao import get_user_by_id
from app.errors import InvalidRequest, register_errors
from app.letters.utils import letter_print_day
from app.models import (
    KEY_TYPE_NORMAL,
    LETTER_TYPE,
    NOTIFICATION_CANCELLED,
    EmailBranding,
    LetterBranding,
    Permission,
    Service,
    ServiceContactList,
)
from app.notifications.process_notifications import (
    persist_notification,
    send_notification_to_queue,
)
from app.schema_validation import validate
from app.schemas import (
    api_key_schema,
    detailed_service_schema,
    email_data_request_schema,
    notification_with_template_schema,
    notifications_filter_schema,
    service_schema,
)
from app.service import statistics
from app.service.send_notification import (
    send_one_off_notification,
    send_pdf_letter_notification,
)
from app.service.send_pdf_letter_schema import send_pdf_letter_request
from app.service.sender import send_notification_to_service_users
from app.service.service_broadcast_settings_schema import (
    service_broadcast_settings_schema,
)
from app.service.service_contact_list_schema import (
    create_service_contact_list_schema,
)
from app.service.service_data_retention_schema import (
    add_service_data_retention_request,
    update_service_data_retention_request,
)
from app.service.service_senders_schema import (
    add_service_email_reply_to_request,
    add_service_letter_contact_block_request,
    add_service_sms_sender_request,
)
from app.service.utils import get_guest_list_objects
from app.user.users_schema import post_set_permissions_schema
from app.utils import (
    DATE_FORMAT,
    DATETIME_FORMAT_NO_TIMEZONE,
    midnight_n_days_ago,
    pagination_links,
)

service_blueprint = Blueprint('service', __name__)

register_errors(service_blueprint)


@service_blueprint.errorhandler(IntegrityError)
def handle_integrity_error(exc):
    """
    Handle integrity errors caused by the unique constraint on ix_organisation_name
    """
    if any(
        'duplicate key value violates unique constraint "{}"'.format(constraint) in str(exc)
        for constraint in {'services_name_key', 'services_email_from_key'}
    ):
        return jsonify(
            result='error',
            message={'name': ["Duplicate service name '{}'".format(
                exc.params.get('name', exc.params.get('email_from', ''))
            )]}
        ), 400
    current_app.logger.exception(exc)
    return jsonify(result='error', message="Internal server error"), 500


@service_blueprint.route('', methods=['GET'])
def get_services():
    only_active = request.args.get('only_active') == 'True'
    detailed = request.args.get('detailed') == 'True'
    user_id = request.args.get('user_id', None)
    include_from_test_key = request.args.get('include_from_test_key', 'True') != 'False'

    # If start and end date are not set, we are expecting today's stats.
    today = str(datetime.utcnow().date())

    start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
    end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()

    if user_id:
        services = dao_fetch_all_services_by_user(user_id, only_active)
    elif detailed:
        result = jsonify(data=get_detailed_services(start_date=start_date, end_date=end_date,
                                                    only_active=only_active,
                                                    include_from_test_key=include_from_test_key
                                                    ))
        return result
    else:
        services = dao_fetch_all_services(only_active)
    data = service_schema.dump(services, many=True).data
    return jsonify(data=data)


@service_blueprint.route('/find-services-by-name', methods=['GET'])
def find_services_by_name():
    service_name = request.args.get('service_name')
    if not service_name:
        errors = {'service_name': ['Missing data for required field.']}
        raise InvalidRequest(errors, status_code=400)
    fetched_services = get_services_by_partial_name(service_name)
    data = [service.serialize_for_org_dashboard() for service in fetched_services]
    return jsonify(data=data), 200


@service_blueprint.route('/live-services-data', methods=['GET'])
def get_live_services_data():
    data = dao_fetch_live_services_data()
    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>', methods=['GET'])
def get_service_by_id(service_id):
    if request.args.get('detailed') == 'True':
        data = get_detailed_service(service_id, today_only=request.args.get('today_only') == 'True')
    else:
        fetched = dao_fetch_service_by_id(service_id)

        data = service_schema.dump(fetched).data
    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>/statistics')
def get_service_notification_statistics(service_id):
    return jsonify(data=get_service_statistics(
        service_id,
        request.args.get('today_only') == 'True',
        int(request.args.get('limit_days', 7))
    ))


@service_blueprint.route('', methods=['POST'])
def create_service():
    data = request.get_json()

    if not data.get('user_id'):
        errors = {'user_id': ['Missing data for required field.']}
        raise InvalidRequest(errors, status_code=400)
    data.pop('service_domain', None)

    # validate json with marshmallow
    service_schema.load(data)

    user = get_user_by_id(data.pop('user_id'))

    # unpack valid json into service object
    valid_service = Service.from_json(data)

    with transaction():
        dao_create_service(valid_service, user)
        set_default_free_allowance_for_service(service=valid_service, year_start=None)

    return jsonify(data=service_schema.dump(valid_service).data), 201


@service_blueprint.route('/<uuid:service_id>', methods=['POST'])
def update_service(service_id):
    req_json = request.get_json()
    fetched_service = dao_fetch_service_by_id(service_id)
    # Capture the status change here as Marshmallow changes this later
    service_going_live = fetched_service.restricted and not req_json.get('restricted', True)
    current_data = dict(service_schema.dump(fetched_service).data.items())
    current_data.update(request.get_json())

    service = service_schema.load(current_data).data

    if 'email_branding' in req_json:
        email_branding_id = req_json['email_branding']
        service.email_branding = None if not email_branding_id else EmailBranding.query.get(email_branding_id)
    if 'letter_branding' in req_json:
        letter_branding_id = req_json['letter_branding']
        service.letter_branding = None if not letter_branding_id else LetterBranding.query.get(letter_branding_id)
    dao_update_service(service)

    if service_going_live:
        send_notification_to_service_users(
            service_id=service_id,
            template_id=current_app.config['SERVICE_NOW_LIVE_TEMPLATE_ID'],
            personalisation={
                'service_name': current_data['name'],
                'message_limit': '{:,}'.format(current_data['message_limit'])
            },
            include_user_fields=['name']
        )

    return jsonify(data=service_schema.dump(fetched_service).data), 200


@service_blueprint.route('/<uuid:service_id>/api-key', methods=['POST'])
def create_api_key(service_id=None):
    fetched_service = dao_fetch_service_by_id(service_id=service_id)
    valid_api_key = api_key_schema.load(request.get_json()).data
    valid_api_key.service = fetched_service
    save_model_api_key(valid_api_key)
    unsigned_api_key = get_unsigned_secret(valid_api_key.id)
    return jsonify(data=unsigned_api_key), 201


@service_blueprint.route('/<uuid:service_id>/api-key/revoke/<uuid:api_key_id>', methods=['POST'])
def revoke_api_key(service_id, api_key_id):
    expire_api_key(service_id=service_id, api_key_id=api_key_id)
    return jsonify(), 202


@service_blueprint.route('/<uuid:service_id>/api-keys', methods=['GET'])
@service_blueprint.route('/<uuid:service_id>/api-keys/<uuid:key_id>', methods=['GET'])
def get_api_keys(service_id, key_id=None):
    dao_fetch_service_by_id(service_id=service_id)

    try:
        if key_id:
            api_keys = [get_model_api_keys(service_id=service_id, id=key_id)]
        else:
            api_keys = get_model_api_keys(service_id=service_id)
    except NoResultFound:
        error = "API key not found for id: {}".format(service_id)
        raise InvalidRequest(error, status_code=404)

    return jsonify(apiKeys=api_key_schema.dump(api_keys, many=True).data), 200


@service_blueprint.route('/<uuid:service_id>/users', methods=['GET'])
def get_users_for_service(service_id):
    fetched = dao_fetch_service_by_id(service_id)
    return jsonify(data=[x.serialize() for x in fetched.users])


@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])
def add_user_to_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)

    if user in service.users:
        error = 'User id: {} already part of service id: {}'.format(user_id, service_id)
        raise InvalidRequest(error, status_code=400)

    data = request.get_json()
    validate(data, post_set_permissions_schema)

    permissions = [
        Permission(service_id=service_id, user_id=user_id, permission=p['permission'])
        for p in data['permissions']
    ]
    folder_permissions = data.get('folder_permissions', [])

    dao_add_user_to_service(service, user, permissions, folder_permissions)
    data = service_schema.dump(service).data
    return jsonify(data=data), 201


@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['DELETE'])
def remove_user_from_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)
    if user not in service.users:
        error = 'User not found'
        raise InvalidRequest(error, status_code=404)

    elif len(service.users) == 1:
        error = 'You cannot remove the only user for a service'
        raise InvalidRequest(error, status_code=400)

    dao_remove_user_from_service(service, user)
    return jsonify({}), 204


# This is placeholder get method until more thought
# goes into how we want to fetch and view various items in history
# tables. This is so product owner can pass stories as done
@service_blueprint.route('/<uuid:service_id>/history', methods=['GET'])
def get_service_history(service_id):
    from app.models import ApiKey, Service, TemplateHistory
    from app.schemas import (
        api_key_history_schema,
        service_history_schema,
        template_history_schema,
    )

    service_history = Service.get_history_model().query.filter_by(id=service_id).all()
    service_data = service_history_schema.dump(service_history, many=True).data
    api_key_history = ApiKey.get_history_model().query.filter_by(service_id=service_id).all()
    api_keys_data = api_key_history_schema.dump(api_key_history, many=True).data

    template_history = TemplateHistory.query.filter_by(service_id=service_id).all()
    template_data, errors = template_history_schema.dump(template_history, many=True)

    data = {
        'service_history': service_data,
        'api_key_history': api_keys_data,
        'template_history': template_data,
        'events': []}

    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>/notifications', methods=['GET'])
def get_all_notifications_for_service(service_id):
    data = notifications_filter_schema.load(request.args).data
    if data.get('to'):
        notification_type = data.get('template_type')[0] if data.get('template_type') else None
        return search_for_notification_by_to_field(service_id=service_id,
                                                   search_term=data['to'],
                                                   statuses=data.get('status'),
                                                   notification_type=notification_type)
    page = data['page'] if 'page' in data else 1
    page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
    limit_days = data.get('limit_days')
    include_jobs = data.get('include_jobs', True)
    include_from_test_key = data.get('include_from_test_key', False)
    include_one_off = data.get('include_one_off', True)

    # count_pages is not being used for whether to count the number of pages, but instead as a flag
    # for whether to show pagination links
    count_pages = data.get('count_pages', True)

    pagination = notifications_dao.get_notifications_for_service(
        service_id,
        filter_dict=data,
        page=page,
        page_size=page_size,
        count_pages=False,
        limit_days=limit_days,
        include_jobs=include_jobs,
        include_from_test_key=include_from_test_key,
        include_one_off=include_one_off
    )

    kwargs = request.args.to_dict()
    kwargs['service_id'] = service_id

    if data.get('format_for_csv'):
        notifications = [notification.serialize_for_csv() for notification in pagination.items]
    else:
        notifications = notification_with_template_schema.dump(pagination.items, many=True).data

    # We try and get the next page of results to work out if we need provide a pagination link to the next page
    # in our response if it exists. Note, this could be done instead by changing `count_pages` in the previous
    # call to be True which will enable us to use Flask-Sqlalchemy to tell if there is a next page of results but
    # this way is much more performant for services with many results (unlike Flask SqlAlchemy, this approach
    # doesn't do an additional query to count all the results of which there could be millions but instead only
    # asks for a single extra page of results).
    next_page_of_pagination = notifications_dao.get_notifications_for_service(
        service_id,
        filter_dict=data,
        page=page + 1,
        page_size=page_size,
        count_pages=False,
        limit_days=limit_days,
        include_jobs=include_jobs,
        include_from_test_key=include_from_test_key,
        include_one_off=include_one_off,
        error_out=False  # False so that if there are no results, it doesn't end in aborting with a 404
    )

    def get_prev_next_pagination_links(current_page, next_page_exists, endpoint, **kwargs):
        if 'page' in kwargs:
            kwargs.pop('page', None)
        links = {}
        if page > 1:
            links['prev'] = url_for(endpoint, page=page - 1, **kwargs)
        if next_page_exists:
            links['next'] = url_for(endpoint, page=page + 1, **kwargs)
        return links

    return jsonify(
        notifications=notifications,
        page_size=page_size,
        links=get_prev_next_pagination_links(
            page,
            len(next_page_of_pagination.items),
            '.get_all_notifications_for_service',
            **kwargs
        ) if count_pages else {}
    ), 200


@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>', methods=['GET'])
def get_notification_for_service(service_id, notification_id):

    notification = notifications_dao.get_notification_with_personalisation(
        service_id,
        notification_id,
        key_type=None,
    )
    return jsonify(
        notification_with_template_schema.dump(notification).data,
    ), 200


@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>/cancel', methods=['POST'])
def cancel_notification_for_service(service_id, notification_id):
    notification = notifications_dao.get_notification_by_id(notification_id, service_id)

    if not notification:
        raise InvalidRequest('Notification not found', status_code=404)
    elif notification.notification_type != LETTER_TYPE:
        raise InvalidRequest('Notification cannot be cancelled - only letters can be cancelled', status_code=400)
    elif not letter_can_be_cancelled(notification.status, notification.created_at):
        print_day = letter_print_day(notification.created_at)
        if too_late_to_cancel_letter(notification.created_at):
            message = "It’s too late to cancel this letter. Printing started {} at 5.30pm".format(print_day)
        elif notification.status == 'cancelled':
            message = "This letter has already been cancelled."
        else:
            message = (
                f"We could not cancel this letter. "
                f"Letter status: {notification.status}, created_at: {notification.created_at}"
            )
        raise InvalidRequest(
            message,
            status_code=400)

    updated_notification = notifications_dao.update_notification_status_by_id(
        notification_id,
        NOTIFICATION_CANCELLED,
    )

    return jsonify(
        notification_with_template_schema.dump(updated_notification).data
    ), 200


def search_for_notification_by_to_field(service_id, search_term, statuses, notification_type):
    results = notifications_dao.dao_get_notifications_by_recipient_or_reference(
        service_id=service_id,
        search_term=search_term,
        statuses=statuses,
        notification_type=notification_type,
        page=1,
        page_size=current_app.config['PAGE_SIZE'],
    )
    return jsonify(
        notifications=notification_with_template_schema.dump(results.items, many=True).data,
        links=pagination_links(
            results,
            '.get_all_notifications_for_service',<commit_msg>Document area that may be doing pagination links when not needed<commit_after>import itertools
from datetime import datetime

from flask import Blueprint, current_app, jsonify, request, url_for
from notifications_utils.letter_timings import (
    letter_can_be_cancelled,
    too_late_to_cancel_letter,
)
from notifications_utils.timezones import convert_utc_to_bst
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound

from app.aws import s3
from app.config import QueueNames
from app.dao import fact_notification_status_dao, notifications_dao
from app.dao.annual_billing_dao import set_default_free_allowance_for_service
from app.dao.api_key_dao import (
    expire_api_key,
    get_model_api_keys,
    get_unsigned_secret,
    save_model_api_key,
)
from app.dao.broadcast_service_dao import set_broadcast_service_type
from app.dao.dao_utils import dao_rollback, transaction
from app.dao.date_util import get_financial_year
from app.dao.fact_notification_status_dao import (
    fetch_monthly_template_usage_for_service,
    fetch_notification_status_for_service_by_month,
    fetch_notification_status_for_service_for_day,
    fetch_notification_status_for_service_for_today_and_7_previous_days,
    fetch_stats_for_all_services_by_date_range,
)
from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
from app.dao.organisation_dao import dao_get_organisation_by_service_id
from app.dao.returned_letters_dao import (
    fetch_most_recent_returned_letter,
    fetch_recent_returned_letter_count,
    fetch_returned_letter_summary,
    fetch_returned_letters,
)
from app.dao.service_contact_list_dao import (
    dao_archive_contact_list,
    dao_get_contact_list_by_id,
    dao_get_contact_lists,
    save_service_contact_list,
)
from app.dao.service_data_retention_dao import (
    fetch_service_data_retention,
    fetch_service_data_retention_by_id,
    fetch_service_data_retention_by_notification_type,
    insert_service_data_retention,
    update_service_data_retention,
)
from app.dao.service_email_reply_to_dao import (
    add_reply_to_email_address_for_service,
    archive_reply_to_email_address,
    dao_get_reply_to_by_id,
    dao_get_reply_to_by_service_id,
    update_reply_to_email_address,
)
from app.dao.service_guest_list_dao import (
    dao_add_and_commit_guest_list_contacts,
    dao_fetch_service_guest_list,
    dao_remove_service_guest_list,
)
from app.dao.service_letter_contact_dao import (
    add_letter_contact_for_service,
    archive_letter_contact,
    dao_get_letter_contact_by_id,
    dao_get_letter_contacts_by_service_id,
    update_letter_contact,
)
from app.dao.service_sms_sender_dao import (
    archive_sms_sender,
    dao_add_sms_sender_for_service,
    dao_get_service_sms_senders_by_id,
    dao_get_sms_senders_by_service_id,
    dao_update_service_sms_sender,
    update_existing_sms_sender_with_inbound_number,
)
from app.dao.services_dao import (
    dao_add_user_to_service,
    dao_archive_service,
    dao_create_service,
    dao_fetch_all_services,
    dao_fetch_all_services_by_user,
    dao_fetch_live_services_data,
    dao_fetch_service_by_id,
    dao_fetch_todays_stats_for_all_services,
    dao_fetch_todays_stats_for_service,
    dao_remove_user_from_service,
    dao_resume_service,
    dao_suspend_service,
    dao_update_service,
    get_services_by_partial_name,
)
from app.dao.templates_dao import dao_get_template_by_id
from app.dao.users_dao import get_user_by_id
from app.errors import InvalidRequest, register_errors
from app.letters.utils import letter_print_day
from app.models import (
    KEY_TYPE_NORMAL,
    LETTER_TYPE,
    NOTIFICATION_CANCELLED,
    EmailBranding,
    LetterBranding,
    Permission,
    Service,
    ServiceContactList,
)
from app.notifications.process_notifications import (
    persist_notification,
    send_notification_to_queue,
)
from app.schema_validation import validate
from app.schemas import (
    api_key_schema,
    detailed_service_schema,
    email_data_request_schema,
    notification_with_template_schema,
    notifications_filter_schema,
    service_schema,
)
from app.service import statistics
from app.service.send_notification import (
    send_one_off_notification,
    send_pdf_letter_notification,
)
from app.service.send_pdf_letter_schema import send_pdf_letter_request
from app.service.sender import send_notification_to_service_users
from app.service.service_broadcast_settings_schema import (
    service_broadcast_settings_schema,
)
from app.service.service_contact_list_schema import (
    create_service_contact_list_schema,
)
from app.service.service_data_retention_schema import (
    add_service_data_retention_request,
    update_service_data_retention_request,
)
from app.service.service_senders_schema import (
    add_service_email_reply_to_request,
    add_service_letter_contact_block_request,
    add_service_sms_sender_request,
)
from app.service.utils import get_guest_list_objects
from app.user.users_schema import post_set_permissions_schema
from app.utils import (
    DATE_FORMAT,
    DATETIME_FORMAT_NO_TIMEZONE,
    midnight_n_days_ago,
    pagination_links,
)

service_blueprint = Blueprint('service', __name__)

register_errors(service_blueprint)


@service_blueprint.errorhandler(IntegrityError)
def handle_integrity_error(exc):
    """
    Handle integrity errors caused by the unique constraint on ix_organisation_name
    """
    if any(
        'duplicate key value violates unique constraint "{}"'.format(constraint) in str(exc)
        for constraint in {'services_name_key', 'services_email_from_key'}
    ):
        return jsonify(
            result='error',
            message={'name': ["Duplicate service name '{}'".format(
                exc.params.get('name', exc.params.get('email_from', ''))
            )]}
        ), 400
    current_app.logger.exception(exc)
    return jsonify(result='error', message="Internal server error"), 500


@service_blueprint.route('', methods=['GET'])
def get_services():
    only_active = request.args.get('only_active') == 'True'
    detailed = request.args.get('detailed') == 'True'
    user_id = request.args.get('user_id', None)
    include_from_test_key = request.args.get('include_from_test_key', 'True') != 'False'

    # If start and end date are not set, we are expecting today's stats.
    today = str(datetime.utcnow().date())

    start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
    end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()

    if user_id:
        services = dao_fetch_all_services_by_user(user_id, only_active)
    elif detailed:
        result = jsonify(data=get_detailed_services(start_date=start_date, end_date=end_date,
                                                    only_active=only_active,
                                                    include_from_test_key=include_from_test_key
                                                    ))
        return result
    else:
        services = dao_fetch_all_services(only_active)
    data = service_schema.dump(services, many=True).data
    return jsonify(data=data)


@service_blueprint.route('/find-services-by-name', methods=['GET'])
def find_services_by_name():
    service_name = request.args.get('service_name')
    if not service_name:
        errors = {'service_name': ['Missing data for required field.']}
        raise InvalidRequest(errors, status_code=400)
    fetched_services = get_services_by_partial_name(service_name)
    data = [service.serialize_for_org_dashboard() for service in fetched_services]
    return jsonify(data=data), 200


@service_blueprint.route('/live-services-data', methods=['GET'])
def get_live_services_data():
    data = dao_fetch_live_services_data()
    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>', methods=['GET'])
def get_service_by_id(service_id):
    if request.args.get('detailed') == 'True':
        data = get_detailed_service(service_id, today_only=request.args.get('today_only') == 'True')
    else:
        fetched = dao_fetch_service_by_id(service_id)

        data = service_schema.dump(fetched).data
    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>/statistics')
def get_service_notification_statistics(service_id):
    return jsonify(data=get_service_statistics(
        service_id,
        request.args.get('today_only') == 'True',
        int(request.args.get('limit_days', 7))
    ))


@service_blueprint.route('', methods=['POST'])
def create_service():
    data = request.get_json()

    if not data.get('user_id'):
        errors = {'user_id': ['Missing data for required field.']}
        raise InvalidRequest(errors, status_code=400)
    data.pop('service_domain', None)

    # validate json with marshmallow
    service_schema.load(data)

    user = get_user_by_id(data.pop('user_id'))

    # unpack valid json into service object
    valid_service = Service.from_json(data)

    with transaction():
        dao_create_service(valid_service, user)
        set_default_free_allowance_for_service(service=valid_service, year_start=None)

    return jsonify(data=service_schema.dump(valid_service).data), 201


@service_blueprint.route('/<uuid:service_id>', methods=['POST'])
def update_service(service_id):
    req_json = request.get_json()
    fetched_service = dao_fetch_service_by_id(service_id)
    # Capture the status change here as Marshmallow changes this later
    service_going_live = fetched_service.restricted and not req_json.get('restricted', True)
    current_data = dict(service_schema.dump(fetched_service).data.items())
    current_data.update(request.get_json())

    service = service_schema.load(current_data).data

    if 'email_branding' in req_json:
        email_branding_id = req_json['email_branding']
        service.email_branding = None if not email_branding_id else EmailBranding.query.get(email_branding_id)
    if 'letter_branding' in req_json:
        letter_branding_id = req_json['letter_branding']
        service.letter_branding = None if not letter_branding_id else LetterBranding.query.get(letter_branding_id)
    dao_update_service(service)

    if service_going_live:
        send_notification_to_service_users(
            service_id=service_id,
            template_id=current_app.config['SERVICE_NOW_LIVE_TEMPLATE_ID'],
            personalisation={
                'service_name': current_data['name'],
                'message_limit': '{:,}'.format(current_data['message_limit'])
            },
            include_user_fields=['name']
        )

    return jsonify(data=service_schema.dump(fetched_service).data), 200


@service_blueprint.route('/<uuid:service_id>/api-key', methods=['POST'])
def create_api_key(service_id=None):
    fetched_service = dao_fetch_service_by_id(service_id=service_id)
    valid_api_key = api_key_schema.load(request.get_json()).data
    valid_api_key.service = fetched_service
    save_model_api_key(valid_api_key)
    unsigned_api_key = get_unsigned_secret(valid_api_key.id)
    return jsonify(data=unsigned_api_key), 201


@service_blueprint.route('/<uuid:service_id>/api-key/revoke/<uuid:api_key_id>', methods=['POST'])
def revoke_api_key(service_id, api_key_id):
    expire_api_key(service_id=service_id, api_key_id=api_key_id)
    return jsonify(), 202


@service_blueprint.route('/<uuid:service_id>/api-keys', methods=['GET'])
@service_blueprint.route('/<uuid:service_id>/api-keys/<uuid:key_id>', methods=['GET'])
def get_api_keys(service_id, key_id=None):
    dao_fetch_service_by_id(service_id=service_id)

    try:
        if key_id:
            api_keys = [get_model_api_keys(service_id=service_id, id=key_id)]
        else:
            api_keys = get_model_api_keys(service_id=service_id)
    except NoResultFound:
        error = "API key not found for id: {}".format(service_id)
        raise InvalidRequest(error, status_code=404)

    return jsonify(apiKeys=api_key_schema.dump(api_keys, many=True).data), 200


@service_blueprint.route('/<uuid:service_id>/users', methods=['GET'])
def get_users_for_service(service_id):
    fetched = dao_fetch_service_by_id(service_id)
    return jsonify(data=[x.serialize() for x in fetched.users])


@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])
def add_user_to_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)

    if user in service.users:
        error = 'User id: {} already part of service id: {}'.format(user_id, service_id)
        raise InvalidRequest(error, status_code=400)

    data = request.get_json()
    validate(data, post_set_permissions_schema)

    permissions = [
        Permission(service_id=service_id, user_id=user_id, permission=p['permission'])
        for p in data['permissions']
    ]
    folder_permissions = data.get('folder_permissions', [])

    dao_add_user_to_service(service, user, permissions, folder_permissions)
    data = service_schema.dump(service).data
    return jsonify(data=data), 201


@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['DELETE'])
def remove_user_from_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)
    if user not in service.users:
        error = 'User not found'
        raise InvalidRequest(error, status_code=404)

    elif len(service.users) == 1:
        error = 'You cannot remove the only user for a service'
        raise InvalidRequest(error, status_code=400)

    dao_remove_user_from_service(service, user)
    return jsonify({}), 204


# This is placeholder get method until more thought
# goes into how we want to fetch and view various items in history
# tables. This is so product owner can pass stories as done
@service_blueprint.route('/<uuid:service_id>/history', methods=['GET'])
def get_service_history(service_id):
    from app.models import ApiKey, Service, TemplateHistory
    from app.schemas import (
        api_key_history_schema,
        service_history_schema,
        template_history_schema,
    )

    service_history = Service.get_history_model().query.filter_by(id=service_id).all()
    service_data = service_history_schema.dump(service_history, many=True).data
    api_key_history = ApiKey.get_history_model().query.filter_by(service_id=service_id).all()
    api_keys_data = api_key_history_schema.dump(api_key_history, many=True).data

    template_history = TemplateHistory.query.filter_by(service_id=service_id).all()
    template_data, errors = template_history_schema.dump(template_history, many=True)

    data = {
        'service_history': service_data,
        'api_key_history': api_keys_data,
        'template_history': template_data,
        'events': []}

    return jsonify(data=data)


@service_blueprint.route('/<uuid:service_id>/notifications', methods=['GET'])
def get_all_notifications_for_service(service_id):
    data = notifications_filter_schema.load(request.args).data
    if data.get('to'):
        notification_type = data.get('template_type')[0] if data.get('template_type') else None
        return search_for_notification_by_to_field(service_id=service_id,
                                                   search_term=data['to'],
                                                   statuses=data.get('status'),
                                                   notification_type=notification_type)
    page = data['page'] if 'page' in data else 1
    page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
    limit_days = data.get('limit_days')
    include_jobs = data.get('include_jobs', True)
    include_from_test_key = data.get('include_from_test_key', False)
    include_one_off = data.get('include_one_off', True)

    # count_pages is not being used for whether to count the number of pages, but instead as a flag
    # for whether to show pagination links
    count_pages = data.get('count_pages', True)

    pagination = notifications_dao.get_notifications_for_service(
        service_id,
        filter_dict=data,
        page=page,
        page_size=page_size,
        count_pages=False,
        limit_days=limit_days,
        include_jobs=include_jobs,
        include_from_test_key=include_from_test_key,
        include_one_off=include_one_off
    )

    kwargs = request.args.to_dict()
    kwargs['service_id'] = service_id

    if data.get('format_for_csv'):
        notifications = [notification.serialize_for_csv() for notification in pagination.items]
    else:
        notifications = notification_with_template_schema.dump(pagination.items, many=True).data

    # We try and get the next page of results to work out if we need provide a pagination link to the next page
    # in our response if it exists. Note, this could be done instead by changing `count_pages` in the previous
    # call to be True which will enable us to use Flask-Sqlalchemy to tell if there is a next page of results but
    # this way is much more performant for services with many results (unlike Flask SqlAlchemy, this approach
    # doesn't do an additional query to count all the results of which there could be millions but instead only
    # asks for a single extra page of results).
    next_page_of_pagination = notifications_dao.get_notifications_for_service(
        service_id,
        filter_dict=data,
        page=page + 1,
        page_size=page_size,
        count_pages=False,
        limit_days=limit_days,
        include_jobs=include_jobs,
        include_from_test_key=include_from_test_key,
        include_one_off=include_one_off,
        error_out=False  # False so that if there are no results, it doesn't end in aborting with a 404
    )

    def get_prev_next_pagination_links(current_page, next_page_exists, endpoint, **kwargs):
        if 'page' in kwargs:
            kwargs.pop('page', None)
        links = {}
        if page > 1:
            links['prev'] = url_for(endpoint, page=page - 1, **kwargs)
        if next_page_exists:
            links['next'] = url_for(endpoint, page=page + 1, **kwargs)
        return links

    return jsonify(
        notifications=notifications,
        page_size=page_size,
        links=get_prev_next_pagination_links(
            page,
            len(next_page_of_pagination.items),
            '.get_all_notifications_for_service',
            **kwargs
        ) if count_pages else {}
    ), 200


@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>', methods=['GET'])
def get_notification_for_service(service_id, notification_id):

    notification = notifications_dao.get_notification_with_personalisation(
        service_id,
        notification_id,
        key_type=None,
    )
    return jsonify(
        notification_with_template_schema.dump(notification).data,
    ), 200


@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>/cancel', methods=['POST'])
def cancel_notification_for_service(service_id, notification_id):
    notification = notifications_dao.get_notification_by_id(notification_id, service_id)

    if not notification:
        raise InvalidRequest('Notification not found', status_code=404)
    elif notification.notification_type != LETTER_TYPE:
        raise InvalidRequest('Notification cannot be cancelled - only letters can be cancelled', status_code=400)
    elif not letter_can_be_cancelled(notification.status, notification.created_at):
        print_day = letter_print_day(notification.created_at)
        if too_late_to_cancel_letter(notification.created_at):
            message = "It’s too late to cancel this letter. Printing started {} at 5.30pm".format(print_day)
        elif notification.status == 'cancelled':
            message = "This letter has already been cancelled."
        else:
            message = (
                f"We could not cancel this letter. "
                f"Letter status: {notification.status}, created_at: {notification.created_at}"
            )
        raise InvalidRequest(
            message,
            status_code=400)

    updated_notification = notifications_dao.update_notification_status_by_id(
        notification_id,
        NOTIFICATION_CANCELLED,
    )

    return jsonify(
        notification_with_template_schema.dump(updated_notification).data
    ), 200


def search_for_notification_by_to_field(service_id, search_term, statuses, notification_type):
    results = notifications_dao.dao_get_notifications_by_recipient_or_reference(
        service_id=service_id,
        search_term=search_term,
        statuses=statuses,
        notification_type=notification_type,
        page=1,
        page_size=current_app.config['PAGE_SIZE'],
    )
    return jsonify(
        notifications=notification_with_template_schema.dump(results.items, many=True).data,
        # TODO: this may be a bug to include the pagination links as currently `search_for_notification_by_to_field`
        # hardcodes the pages of results to always be the first page so not sure what benefit is to show a link to
        # page 2 which would have the page parameter ignored
        links=pagination_links(
            results,
            '.get_all_notifications_for_service',
{'fraction_non_alphanumeric': 0.08572645124662913, 'fraction_numerical': 0.003950418697071486, 'mean_word_length': 4.063480656366032, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '874641', 'n_tokens_mistral': 13302, 'n_tokens_neox': 13040, 'n_words': 2414}