<commit_before>"""
Bridge to xBrowse for management commands like loading projects, etc
Celery tasks are just shells over these functions

Make sure DJANGO_SETTINGS_MODULE is set before calling - this script gets entities from django.conf

***
Future Plan
***
reload_exome_coverage() -> kwargs family, project
reload_variants() -> kwargs family, project

"""

import os
import gzip
import shutil
import vcf

from datetime import datetime, date
from django.conf import settings
from xbrowse_server import mall

from xbrowse_server.base.models import Project, Individual, Family, Cohort
from xbrowse import genomeloc
from xbrowse_server.mall import get_mall, get_cnv_store, get_coverage_store, get_project_datastore


def clean_project(project_id):
    """
    Clear data for this project from all the xbrowse resources:
     - datastore
     - coverage store
     - cnv store
    Does not remove any of the project's data links - so no data is lost, but everything must be rebuilt
    """
    project = Project.objects.get(project_id=project_id)
    individuals = project.get_individuals()

    # datastore
    get_mall(project_id).variant_store.delete_project(project_id)

    # coverage store
    for individual in individuals:
        get_coverage_store().remove_sample(individual.get_coverage_store_id())

    # cnv store
    for individual in individuals:
        get_cnv_store().remove_sample(individual.get_coverage_store_id())


def load_project(project_id, force_annotations=False, vcf_files=None, start_from_chrom=None, end_with_chrom=None):
    """
    Reload a whole project
    """
    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- starting load_project: " + project_id))
    project = Project.objects.get(project_id=project_id)

    #load_project_coverage(project_id)
    if vcf_files is None:
        load_project_variants(project_id, force_annotations=force_annotations, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)
    else:
        load_project_variants_from_vcf(project_id, vcf_files=vcf_files, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)

    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- load_project: " + project_id + " is done!"))




def load_coverage_for_individuals(individuals):
    for individual in individuals:
        if individual.coverage_file:
            get_coverage_store().add_sample(individual.get_coverage_store_id(), gzip.open(individual.coverage_file))<commit_msg>When variants are done loading, convert all family' status from 'Waiting for data' to 'Analysis in Progress'<commit_after>"""
Bridge to xBrowse for management commands like loading projects, etc
Celery tasks are just shells over these functions

Make sure DJANGO_SETTINGS_MODULE is set before calling - this script gets entities from django.conf

***
Future Plan
***
reload_exome_coverage() -> kwargs family, project
reload_variants() -> kwargs family, project

"""

import os
import gzip
import shutil
import vcf

from datetime import datetime, date
from django.conf import settings
from xbrowse_server import mall

from xbrowse_server.base.models import Project, Individual, Family, Cohort
from xbrowse import genomeloc
from xbrowse_server.mall import get_mall, get_cnv_store, get_coverage_store, get_project_datastore


def clean_project(project_id):
    """
    Clear data for this project from all the xbrowse resources:
     - datastore
     - coverage store
     - cnv store
    Does not remove any of the project's data links - so no data is lost, but everything must be rebuilt
    """
    project = Project.objects.get(project_id=project_id)
    individuals = project.get_individuals()

    # datastore
    get_mall(project_id).variant_store.delete_project(project_id)

    # coverage store
    for individual in individuals:
        get_coverage_store().remove_sample(individual.get_coverage_store_id())

    # cnv store
    for individual in individuals:
        get_cnv_store().remove_sample(individual.get_coverage_store_id())


def load_project(project_id, force_annotations=False, vcf_files=None, start_from_chrom=None, end_with_chrom=None):
    """
    Reload a whole project
    """
    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- starting load_project: " + project_id))
    project = Project.objects.get(project_id=project_id)

    load_project_coverage(project_id)
    if vcf_files is None:
        load_project_variants(project_id, force_annotations=force_annotations, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)
    else:
        load_project_variants_from_vcf(project_id, vcf_files=vcf_files, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)

    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- load_project: " + project_id + " is done!"))

    # update the analysis status from 'Waiting for data' to 'Analysis in Progress'
    for f in Family.objects.filter(project__project_id=project_id):
        if f.analysis_status == 'Q':
            f.analysis_status = 'I'
            f.save()



def load_coverage_for_individuals(individuals):
    for individual in individuals:
        if individual.coverage_file:
            get_coverage_store().add_sample(individual.get_coverage_store_id(), gzip.open(individual.coverage_file))
{'fraction_non_alphanumeric': 0.084547002080575, 'fraction_numerical': 0.0, 'mean_word_length': 4.595767195767196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3610487', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1604, 'n_words': 409}