<commit_before>import json
import logging
from collections import defaultdict
from django.contrib.auth.models import User

from seqr.models import SavedVariant, VariantSearchResults, Individual
from seqr.utils.es_utils import get_es_variants_for_variant_tuples, InvalidIndexException
from seqr.utils.xpos_utils import get_chrom_pos


from xbrowse_server.api.utils import add_extra_info_to_variants_project
from xbrowse_server.mall import get_reference
from xbrowse_server.base.models import Project as BaseProject
from xbrowse_server.base.lookups import get_variants_from_variant_tuples


def deprecated_get_or_create_saved_variant(xpos=None, ref=None, alt=None, family=None, project=None, **kwargs):
    if not project:
        project = family.project
    saved_variant, _ = SavedVariant.objects.get_or_create(
        xpos_start=xpos,
        xpos_end=xpos + len(ref) - 1,
        ref=ref,
        alt=alt,
        family=family,
        project=project,
    )
    if not saved_variant.saved_variant_json:
        try:
            saved_variants_json = _retrieve_saved_variants_json(project, [(xpos, ref, alt, family)], create_if_missing=True)
            if len(saved_variants_json):
                _update_saved_variant_json(saved_variant, saved_variants_json[0])
        except Exception as e:
            logging.error("Unable to retrieve variant annotations for %s (%s, %s, %s): %s" % (family, xpos, ref, alt, e))
    return saved_variant


def update_project_saved_variant_json(project, family_id=None):
    saved_variants = SavedVariant.objects.filter(project=project, family__isnull=False).select_related('family')
    if family_id:
        saved_variants = saved_variants.filter(family__family_id=family_id)

    saved_variants_map = {(v.xpos_start, v.ref, v.alt, v.family): v for v in saved_variants}
    variant_tuples = saved_variants_map.keys()
    saved_variants_map = {
        (xpos, ref, alt, family.guid): v for (xpos, ref, alt, family), v in saved_variants_map.items()
    }

    variants_json = _retrieve_saved_variants_json(project, variant_tuples)

    updated_saved_variant_guids = []
    for var in variants_json:
        for family_guid in var['familyGuids']:
            saved_variant = saved_variants_map.get((var['xpos'], var['ref'], var['alt'], family_guid))
            if saved_variant:
                _update_saved_variant_json(saved_variant, var)
                updated_saved_variant_guids.append(saved_variant.guid)

    return updated_saved_variant_guids


def reset_cached_search_results(project):
    # TODO fix this
    results = VariantSearchResults.objects.filter(results__isnull=False)
    if project:
        results = VariantSearchResults.objects.filter(families__project=project)
    results.distinct().update(
        results=None,
        total_results=None,
    )


def _retrieve_saved_variants_json(project, variant_tuples, create_if_missing=False):
    xpos_ref_alt_tuples = []
    xpos_ref_alt_family_tuples = []
    family_id_to_guid = {}
    for xpos, ref, alt, family in variant_tuples:
        xpos_ref_alt_tuples.append((xpos, ref, alt))
        xpos_ref_alt_family_tuples.append((xpos, ref, alt, family.family_id))
        family_id_to_guid[family.family_id] = family.guid

    try:
        families = project.family_set.filter(guid__in=family_id_to_guid.values())
        return get_es_variants_for_variant_tuples(families, xpos_ref_alt_tuples)
    except InvalidIndexException:
        variants = _deprecated_retrieve_saved_variants_json(project, xpos_ref_alt_family_tuples, create_if_missing)<commit_msg>update reset for cached results<commit_after>import json
import logging
import redis
from collections import defaultdict
from django.contrib.auth.models import User

from seqr.models import SavedVariant, VariantSearchResults, Individual
from seqr.utils.es_utils import get_es_variants_for_variant_tuples, InvalidIndexException
from seqr.utils.xpos_utils import get_chrom_pos
from settings import REDIS_SERVICE_HOSTNAME

from xbrowse_server.api.utils import add_extra_info_to_variants_project
from xbrowse_server.mall import get_reference
from xbrowse_server.base.models import Project as BaseProject
from xbrowse_server.base.lookups import get_variants_from_variant_tuples

logger = logging.getLogger(__name__)


def deprecated_get_or_create_saved_variant(xpos=None, ref=None, alt=None, family=None, project=None, **kwargs):
    if not project:
        project = family.project
    saved_variant, _ = SavedVariant.objects.get_or_create(
        xpos_start=xpos,
        xpos_end=xpos + len(ref) - 1,
        ref=ref,
        alt=alt,
        family=family,
        project=project,
    )
    if not saved_variant.saved_variant_json:
        try:
            saved_variants_json = _retrieve_saved_variants_json(project, [(xpos, ref, alt, family)], create_if_missing=True)
            if len(saved_variants_json):
                _update_saved_variant_json(saved_variant, saved_variants_json[0])
        except Exception as e:
            logger.error("Unable to retrieve variant annotations for %s (%s, %s, %s): %s" % (family, xpos, ref, alt, e))
    return saved_variant


def update_project_saved_variant_json(project, family_id=None):
    saved_variants = SavedVariant.objects.filter(project=project, family__isnull=False).select_related('family')
    if family_id:
        saved_variants = saved_variants.filter(family__family_id=family_id)

    saved_variants_map = {(v.xpos_start, v.ref, v.alt, v.family): v for v in saved_variants}
    variant_tuples = saved_variants_map.keys()
    saved_variants_map = {
        (xpos, ref, alt, family.guid): v for (xpos, ref, alt, family), v in saved_variants_map.items()
    }

    variants_json = _retrieve_saved_variants_json(project, variant_tuples)

    updated_saved_variant_guids = []
    for var in variants_json:
        for family_guid in var['familyGuids']:
            saved_variant = saved_variants_map.get((var['xpos'], var['ref'], var['alt'], family_guid))
            if saved_variant:
                _update_saved_variant_json(saved_variant, var)
                updated_saved_variant_guids.append(saved_variant.guid)

    return updated_saved_variant_guids


def reset_cached_search_results(project):
    try:
        redis_client = redis.StrictRedis(host=REDIS_SERVICE_HOSTNAME, socket_connect_timeout=3)
        keys_to_delete = []
        if project:
            result_guids = [res.guid for res in VariantSearchResults.objects.filter(families__project=project)]
            for guid in result_guids:
                keys_to_delete += redis_client.scan_iter(match='search_results__{}*'.format(guid))
        else:
            keys_to_delete = redis_client.keys(pattern='search_results__*')
        redis_client.delete(*keys_to_delete)
        logger.info('Reset {} cached results'.format(len(keys_to_delete)))
    except Exception as e:
        logger.error("Unable to reset cached search results: {}".format(e))


def _retrieve_saved_variants_json(project, variant_tuples, create_if_missing=False):
    xpos_ref_alt_tuples = []
    xpos_ref_alt_family_tuples = []
    family_id_to_guid = {}
    for xpos, ref, alt, family in variant_tuples:
        xpos_ref_alt_tuples.append((xpos, ref, alt))
        xpos_ref_alt_family_tuples.append((xpos, ref, alt, family.family_id))
        family_id_to_guid[family.family_id] = family.guid

    try:
        families = project.family_set.filter(guid__in=family_id_to_guid.values())
        return get_es_variants_for_variant_tuples(families, xpos_ref_alt_tuples)
    except InvalidIndexException:
        variants = _deprecated_retrieve_saved_variants_json(project, xpos_ref_alt_family_tuples, create_if_missing)
{'fraction_non_alphanumeric': 0.0839078964485495, 'fraction_numerical': 0.0006504488096786782, 'mean_word_length': 4.128752501667779, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4131462', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2419, 'n_words': 462}