<commit_before>"""
seqr integrates with the `PhenoTips <http://phenotips.org>`_ UI so that users can enter
detailed phenotype information for individuals.
The PhenoTips server is installed locally so that it's not visible to the outside network, and
seqr then acts as a proxy for all HTTP requests between the PhenoTips web-based UI (which is
running in users' browser), and the PhenoTips server (running on the internal network).

This proxy setup allows seqr to check authentication and authorization before allowing users to
access patients in PhenoTips, and is similar to how seqr manages access to the SQL database and
other internal systems.

This module implements the proxy functionality + methods for making requests to PhenoTips HTTP APIs.

PhenoTips API docs are at:

https://phenotips.org/DevGuide/API
https://phenotips.org/DevGuide/RESTfulAPI
https://phenotips.org/DevGuide/PermissionsRESTfulAPI
"""

import json
import logging
import re
import requests
from collections import defaultdict

import settings

from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from django.core.exceptions import ObjectDoesNotExist

from reference_data.models import HumanPhenotypeOntology
from seqr.model_utils import update_seqr_model
from seqr.models import Project, CAN_EDIT, CAN_VIEW, Individual
from seqr.views.apis.auth_api import API_LOGIN_REQUIRED_URL
from seqr.views.utils.file_utils import save_uploaded_file
from seqr.views.utils.json_utils import create_json_response
from seqr.views.utils.permissions_utils import check_permissions, get_project_and_check_permissions
from seqr.views.utils.proxy_request_utils import proxy_request

logger = logging.getLogger(__name__)

PHENOTIPS_QUICK_SAVE_URL_REGEX = "/bin/preview/data/(P[0-9]{1,20})"

DO_NOT_PROXY_URL_KEYWORDS = [
    '/delete',
    '/logout',
    '/login',
    '/admin',
    '/CreatePatientRecord',
    '/bin/PatientAccessRightsManagement',
    '/ForgotUsername',
    '/ForgotPassword',
]

FAMILY_ID_COLUMN = 'family_id'
INDIVIDUAL_ID_COLUMN = 'external_id'
HPO_TERMS_PRESENT_COLUMN = 'hpo_present'
HPO_TERMS_ABSENT_COLUMN = 'hpo_absent'
HPO_TERM_NUMBER_COLUMN = 'hpo_number'
AFFECTED_COLUMN = 'affected'
FEATURES_COLUMN = 'features'


@login_required(login_url=API_LOGIN_REQUIRED_URL)
@csrf_exempt
def receive_hpo_table_handler(request, project_guid):
    """Handler for bulk update of hpo terms. This handler parses the records, but doesn't save them in the database.
    Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client.

    Args:
        request (object): Django request object
        project_guid (string): project GUID
    """

    project = get_project_and_check_permissions(project_guid, request.user)

    try:
        uploaded_file_id, _, json_records = save_uploaded_file(request, process_records=_process_hpo_records)
    except Exception as e:
        return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))

    updates_by_individual_guid = {}
    missing_individuals = []
    unchanged_individuals = []
    all_hpo_terms = set()
    for record in json_records:
        family_id = record.get(FAMILY_ID_COLUMN, None)
        individual_id = record.get(INDIVIDUAL_ID_COLUMN)
        individual_q = Individual.objects.filter(
            individual_id__in=[individual_id, '{}_{}'.format(family_id, individual_id)],
            family__project=project,
        )
        if family_id:
            individual_q = individual_q.filter(family__family_id=family_id)
        individual = individual_q.first()
        if individual:
            features = record.get(FEATURES_COLUMN) or []
            if individual.phenotips_data and features and \
                    _feature_set(features) == _feature_set(json.loads(individual.phenotips_data).get('features', [])):
                unchanged_individuals.append(individual_id)
            else:
                all_hpo_terms.update([feature['id'] for feature in features])
                updates_by_individual_guid[individual.guid] = features
        else:
            missing_individuals.append(individual_id)

    if not updates_by_individual_guid:
        return create_json_response({
            'errors': ['Unable to find individuals to update for any of the {total} parsed individuals.{missing}{unchanged}'.format(
                total=len(missing_individuals) + len(unchanged_individuals),
                missing=' No matching ids found for {} individuals'.format(len(missing_individuals)) if missing_individuals else '',
                unchanged=' No changes detected for {} individuals'.format(len(unchanged_individuals)) if unchanged_individuals else '',
            )],
            'warnings': []
        }, status=400, reason='Unable to find any matching individuals')

    info = ['{} individuals will be updated'.format(len(updates_by_individual_guid))]
    warnings = []
    if missing_individuals:
        warnings.append(
            'Unable to find matching ids for {} individuals. The following entries will not be updated: {}'.format(
                len(missing_individuals), ', '.join(missing_individuals)
            ))
    if unchanged_individuals:
        warnings.append(
            'No changes detected for {} individuals. The following entries will not be updated: {}'.format(
                len(unchanged_individuals), ', '.join(unchanged_individuals)
            ))

    hpo_terms = {hpo.hpo_id: hpo for hpo in HumanPhenotypeOntology.objects.filter(hpo_id__in=all_hpo_terms)}
    invalid_hpo_terms = set()
    for features in updates_by_individual_guid.values():
        for feature in features:
            hpo_data = hpo_terms.get(feature['id'])
            if hpo_data:
                feature['category'] = hpo_data.category_id
                feature['label'] = hpo_data.name
            else:
                invalid_hpo_terms.add(feature['id'])
    if invalid_hpo_terms:
        warnings.append(
            "The following HPO terms were not found in seqr's HPO data, and while they will be added they may be incorrect: {}".format(
                ', '.join(invalid_hpo_terms)
            ))

    response = {
        'updatesByIndividualGuid': updates_by_individual_guid,
        'uploadedFileId': uploaded_file_id,
        'errors': [],
        'warnings': warnings,
        'info': info,
    }
    return create_json_response(response)


def _process_hpo_records(records, filename=''):
    if filename.endswith('.json'):
        return records

    column_map = {}
    for i, field in enumerate(records[0]):
        key = field.lower()
        if 'family' in key or 'pedigree' in key:<commit_msg>do not allow invalid HPO term upload<commit_after>"""
seqr integrates with the `PhenoTips <http://phenotips.org>`_ UI so that users can enter
detailed phenotype information for individuals.
The PhenoTips server is installed locally so that it's not visible to the outside network, and
seqr then acts as a proxy for all HTTP requests between the PhenoTips web-based UI (which is
running in users' browser), and the PhenoTips server (running on the internal network).

This proxy setup allows seqr to check authentication and authorization before allowing users to
access patients in PhenoTips, and is similar to how seqr manages access to the SQL database and
other internal systems.

This module implements the proxy functionality + methods for making requests to PhenoTips HTTP APIs.

PhenoTips API docs are at:

https://phenotips.org/DevGuide/API
https://phenotips.org/DevGuide/RESTfulAPI
https://phenotips.org/DevGuide/PermissionsRESTfulAPI
"""

import json
import logging
import re
import requests
from collections import defaultdict

import settings

from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from django.core.exceptions import ObjectDoesNotExist

from reference_data.models import HumanPhenotypeOntology
from seqr.model_utils import update_seqr_model
from seqr.models import Project, CAN_EDIT, CAN_VIEW, Individual
from seqr.views.apis.auth_api import API_LOGIN_REQUIRED_URL
from seqr.views.utils.file_utils import save_uploaded_file
from seqr.views.utils.json_utils import create_json_response
from seqr.views.utils.permissions_utils import check_permissions, get_project_and_check_permissions
from seqr.views.utils.proxy_request_utils import proxy_request

logger = logging.getLogger(__name__)

PHENOTIPS_QUICK_SAVE_URL_REGEX = "/bin/preview/data/(P[0-9]{1,20})"

DO_NOT_PROXY_URL_KEYWORDS = [
    '/delete',
    '/logout',
    '/login',
    '/admin',
    '/CreatePatientRecord',
    '/bin/PatientAccessRightsManagement',
    '/ForgotUsername',
    '/ForgotPassword',
]

FAMILY_ID_COLUMN = 'family_id'
INDIVIDUAL_ID_COLUMN = 'external_id'
HPO_TERMS_PRESENT_COLUMN = 'hpo_present'
HPO_TERMS_ABSENT_COLUMN = 'hpo_absent'
HPO_TERM_NUMBER_COLUMN = 'hpo_number'
AFFECTED_COLUMN = 'affected'
FEATURES_COLUMN = 'features'


@login_required(login_url=API_LOGIN_REQUIRED_URL)
@csrf_exempt
def receive_hpo_table_handler(request, project_guid):
    """Handler for bulk update of hpo terms. This handler parses the records, but doesn't save them in the database.
    Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client.

    Args:
        request (object): Django request object
        project_guid (string): project GUID
    """

    project = get_project_and_check_permissions(project_guid, request.user)

    try:
        uploaded_file_id, _, json_records = save_uploaded_file(request, process_records=_process_hpo_records)
    except Exception as e:
        return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))

    updates_by_individual_guid = {}
    missing_individuals = []
    unchanged_individuals = []
    all_hpo_terms = set()
    for record in json_records:
        family_id = record.get(FAMILY_ID_COLUMN, None)
        individual_id = record.get(INDIVIDUAL_ID_COLUMN)
        individual_q = Individual.objects.filter(
            individual_id__in=[individual_id, '{}_{}'.format(family_id, individual_id)],
            family__project=project,
        )
        if family_id:
            individual_q = individual_q.filter(family__family_id=family_id)
        individual = individual_q.first()
        if individual:
            features = record.get(FEATURES_COLUMN) or []
            if individual.phenotips_data and features and \
                    _feature_set(features) == _feature_set(json.loads(individual.phenotips_data).get('features', [])):
                unchanged_individuals.append(individual_id)
            else:
                all_hpo_terms.update([feature['id'] for feature in features])
                updates_by_individual_guid[individual.guid] = features
        else:
            missing_individuals.append(individual_id)

    if not updates_by_individual_guid:
        return create_json_response({
            'errors': ['Unable to find individuals to update for any of the {total} parsed individuals.{missing}{unchanged}'.format(
                total=len(missing_individuals) + len(unchanged_individuals),
                missing=' No matching ids found for {} individuals'.format(len(missing_individuals)) if missing_individuals else '',
                unchanged=' No changes detected for {} individuals'.format(len(unchanged_individuals)) if unchanged_individuals else '',
            )],
            'warnings': []
        }, status=400, reason='Unable to find any matching individuals')

    hpo_terms = {hpo.hpo_id: hpo for hpo in HumanPhenotypeOntology.objects.filter(hpo_id__in=all_hpo_terms)}
    invalid_hpo_terms = set()
    for features in updates_by_individual_guid.values():
        for feature in features:
            hpo_data = hpo_terms.get(feature['id'])
            if hpo_data:
                feature['category'] = hpo_data.category_id
                feature['label'] = hpo_data.name
            else:
                invalid_hpo_terms.add(feature['id'])
    if invalid_hpo_terms:
        return create_json_response({
            'errors': [
                "The following HPO terms were not found in seqr's HPO data: {}".format(', '.join(invalid_hpo_terms))
            ],
            'warnings': []
        }, status=400, reason='Invalid HPO terms')

    info = ['{} individuals will be updated'.format(len(updates_by_individual_guid))]
    warnings = []
    if missing_individuals:
        warnings.append(
            'Unable to find matching ids for {} individuals. The following entries will not be updated: {}'.format(
                len(missing_individuals), ', '.join(missing_individuals)
            ))
    if unchanged_individuals:
        warnings.append(
            'No changes detected for {} individuals. The following entries will not be updated: {}'.format(
                len(unchanged_individuals), ', '.join(unchanged_individuals)
            ))

    response = {
        'updatesByIndividualGuid': updates_by_individual_guid,
        'uploadedFileId': uploaded_file_id,
        'errors': [],
        'warnings': warnings,
        'info': info,
    }
    return create_json_response(response)


def _process_hpo_records(records, filename=''):
    if filename.endswith('.json'):
        return records

    column_map = {}
    for i, field in enumerate(records[0]):
        key = field.lower()
        if 'family' in key or 'pedigree' in key:
{'fraction_non_alphanumeric': 0.08709346615880457, 'fraction_numerical': 0.001977732200410196, 'mean_word_length': 3.4719947592531937, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '797413', 'n_tokens_mistral': 4117, 'n_tokens_neox': 3917, 'n_words': 1036}