<commit_before>from bson import json_util
import json
import logging
import pymongo
from tqdm import tqdm
import settings

from django.core.management.base import BaseCommand
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q
from guardian.shortcuts import assign_perm

from seqr.utils.model_sync_utils import convert_html_to_plain_text
from seqr.views.utils.variant_utils import get_or_create_saved_variant
from seqr.views.apis import phenotips_api
from seqr.views.apis.phenotips_api import phenotips_patient_exists, _get_patient_data, _update_individual_phenotips_data
from xbrowse_server.base.models import \
    Project, \
    Family, \
    Individual, \
    VariantNote, \
    ProjectTag, \
    VariantTag, \
    VariantFunctionalData, \
    ProjectCollaborator, VCFFile, AnalysedBy

from seqr.models import \
    Project as SeqrProject, \
    Family as SeqrFamily, \
    Individual as SeqrIndividual, \
    VariantTagType as SeqrVariantTagType, \
    VariantTag as SeqrVariantTag, \
    VariantNote as SeqrVariantNote, \
    VariantFunctionalData as SeqrVariantFunctionalData, \
    Sample as SeqrSample, \
    LocusList, \
    FamilyAnalysedBy as SeqrAnalysedBy, \
    CAN_VIEW, ModelWithGUID

from xbrowse_server.mall import get_datastore, get_annotator

logger = logging.getLogger(__name__)

# switching to python3.6 will make this unnecessary as built-in python dictionaries will be ordered
from collections import OrderedDict, defaultdict
class OrderedDefaultDict(OrderedDict, defaultdict):
    def __init__(self, default_factory=None, *args, **kwargs):
        super(OrderedDefaultDict, self).__init__(*args, **kwargs)
        self.default_factory = default_factory


DEBUG = False   # whether to ask before updating values


class Command(BaseCommand):
    help = 'Transfer projects to the new seqr schema'

    def add_arguments(self, parser):
        parser.add_argument('--reset-all-models', help='This flag causes all records to be cleared from the seqr schema\'s Project, Family, and Individual models before transferring data', action='store_true')
        parser.add_argument('--dont-connect-to-phenotips', help='dont retrieve phenotips internal id and latest data', action='store_true')
        parser.add_argument('project_id', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.')

    def handle(self, *args, **options):
        """transfer project"""
        reset_all_models = options['reset_all_models']
        connect_to_phenotips = not options['dont_connect_to_phenotips']
        project_ids_to_process = options['project_id']
        
        counters = OrderedDefaultDict(int)

        #if reset_all_models:
        #    print("Dropping all records from SeqrProject, SeqrFamily, SeqrIndividual")
        #    SeqrIndividual.objects.all().delete()
        #    SeqrFamily.objects.all().delete()
        #    SeqrProject.objects.all().delete()

        # reset models that'll be regenerated
        #if not project_ids_to_process:
        #    SeqrSample.objects.all().delete()
        #    SeqrVariantTagType.objects.all().delete()
        #    SeqrVariantTag.objects.all().delete()
        #    SeqrVariantNote.objects.all().delete()



        if project_ids_to_process:
            projects = Project.objects.filter(project_id__in=project_ids_to_process)
            logging.info("Processing %s projects" % len(projects))
        else:
            projects = Project.objects.filter(
                ~Q(project_id__contains="DEPRECATED") &
                ~Q(project_name__contains="DEPRECATED") &
                ~Q(project_id__istartswith="temp") &
                ~Q(project_id__istartswith="test_") &
                ~Q(project_name__contains="DISABLED_")
            )
            logging.info("Processing all %s projects" % len(projects))
            project_ids_to_process = [p.project_id for p in projects]


        updated_seqr_project_guids = set()
        updated_seqr_family_guids = set()
        updated_seqr_individual_guids = set()

        for source_project in tqdm(projects, unit=" projects"):
            counters['source_projects'] += 1

            print("Project: " + source_project.project_id)

            # transfer Project data
            new_project, project_created = transfer_project(source_project)
            updated_seqr_project_guids.add(new_project.guid)
            if project_created: counters['projects_created'] += 1

            # transfer Families and Individuals
            source_family_id_to_new_family = {}
            for source_family in Family.objects.filter(project=source_project):
                new_family, family_created = transfer_family(source_family, new_project)

                updated_seqr_family_guids.add(new_family.guid)

                if family_created: counters['families_created'] += 1

                source_family_id_to_new_family[source_family.id] = new_family

                for source_individual in Individual.objects.filter(family=source_family):

                    new_individual, individual_created, phenotips_data_retrieved = transfer_individual(
                        source_individual, new_family, new_project, connect_to_phenotips
                    )

                    updated_seqr_individual_guids.add(new_individual.guid)

                    if individual_created: counters['individuals_created'] += 1
                    if phenotips_data_retrieved: counters['individuals_data_retrieved_from_phenotips'] += 1

                    if source_individual.combined_individuals_info:
                        combined_individuals_info = json.loads(source_individual.combined_individuals_info)
                        """
                        combined_individuals_info json is expected to look like:
                        {
                            'WES' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                            'WGS' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                            'RNA' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                        }
                        """
                        for i, sample_type_i, combined_individuals_info_i in enumerate(combined_individuals_info.items()):
                            source_project_i = Project.objects.get(project_id=combined_individuals_info_i['project_id'])
                            #source_family_i = Project.objects.get(project_id=combined_individuals_info_i['family_id'])
                            source_individual_i = Project.objects.get(project_id=combined_individuals_info_i['indiv_id'])

                            create_sample_records(sample_type_i, source_individual_i, new_project, new_individual, counters)
                    else:
                        create_sample_records(source_individual, new_project, new_individual, counters)
                        #combined_families_info.update({from_project_datatype: {'project_id': from_project.project_id, 'family_id': from_f.family_id}})

                for source_analysed_by in AnalysedBy.objects.filter(family=source_family):
                    new_analysed_by, analysed_by_created = transfer_analysed_by(source_analysed_by, new_family)

                    if analysed_by_created: counters['analysed_by_created'] += 1

            # TODO family groups, cohorts
            for source_variant_tag_type in ProjectTag.objects.filter(project=source_project).order_by('order'):
                new_variant_tag_type, created = get_or_create_variant_tag_type(
                    source_variant_tag_type, new_project)

                for source_variant_tag in VariantTag.objects.filter(project_tag=source_variant_tag_type):
                    new_family = source_family_id_to_new_family.get(source_variant_tag.family.id if source_variant_tag.family else None)
                    new_variant_tag, variant_tag_created = get_or_create_variant_tag(
                        source_variant_tag,
                        new_project,
                        new_family,
                        new_variant_tag_type,
                    )

                    if variant_tag_created: counters['variant_tags_created'] += 1

            for source_variant_functional_data in VariantFunctionalData.objects.filter(family__project=source_project):
                new_family = source_family_id_to_new_family.get(source_variant_functional_data.family.id)

                _, variant_functional_data_created = get_or_create_variant_functional_data(
                    source_variant_functional_data,
                    new_project,
                    new_family,
                )

                if variant_functional_data_created:   counters['variant_functional_data_created'] += 1

            for source_variant_note in VariantNote.objects.filter(project=source_project):
                new_family = source_family_id_to_new_family.get(source_variant_note.family.id if source_variant_note.family else None)

                new_variant_note, variant_note_created = get_or_create_variant_note(
                    source_variant_note,
                    new_project,
                    new_family,
                )

                if variant_note_created:   counters['variant_notes_created'] += 1


        for deprecated_project_id in project_ids_to_process:

            base_project = Project.objects.get(project_id=deprecated_project_id)

            # delete Tag type
            for seqr_variant_tag_type in SeqrVariantTagType.objects.filter(project__deprecated_project_id=deprecated_project_id):
                if not ProjectTag.objects.filter(
                    project=base_project,
                    tag=seqr_variant_tag_type.name,
                    title=seqr_variant_tag_type.description,
                    color=seqr_variant_tag_type.color,
                    order=seqr_variant_tag_type.order
                ):
                    seqr_variant_tag_type.delete()
                    print("--- deleting variant tag type: " + str(seqr_variant_tag_type))
                    counters['seqr_variant_tag_type_deleted'] += 1

            # delete Tag
            delete_count, _ = SeqrVariantTag.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_tag_deleted'] += delete_count
            for seqr_variant_tag in SeqrVariantTag.objects.filter(saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantTag.objects.filter(
                        project_tag__project=base_project,
                        project_tag__tag=seqr_variant_tag.variant_tag_type.name,
                        #project_tag__title=seqr_variant_tag.variant_tag_type.description,
                        xpos=seqr_variant_tag.saved_variant.xpos_start,
                        ref=seqr_variant_tag.saved_variant.ref,
                        alt=seqr_variant_tag.saved_variant.alt,
                ):
                    seqr_variant_tag.delete()
                    print("--- deleting variant tag: " + str(seqr_variant_tag))
                    counters['seqr_variant_tag_deleted'] += 1

            # delete functional data tags
            delete_count, _ = SeqrVariantFunctionalData.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_functional_data_deleted'] += delete_count
            for seqr_variant_functional_data in SeqrVariantFunctionalData.objects.filter(
                    saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantFunctionalData.objects.filter(
                        family__project=base_project,
                        functional_data_tag=seqr_variant_functional_data.functional_data_tag,
                        metadata=seqr_variant_functional_data.metadata,
                        xpos=seqr_variant_functional_data.saved_variant.xpos_start,
                        ref=seqr_variant_functional_data.saved_variant.ref,
                        alt=seqr_variant_functional_data.saved_variant.alt,
                ):
                    seqr_variant_functional_data.delete()
                    print("--- deleting variant tag: " + str(seqr_variant_functional_data))
                    counters['seqr_variant_tag_deleted'] += 1

            # delete Variant Note
            delete_count, _ = SeqrVariantNote.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_note_deleted'] += delete_count
            for seqr_variant_note in SeqrVariantNote.objects.filter(saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantNote.objects.filter(
                    project=base_project,
                    note=seqr_variant_note.note,
                    xpos=seqr_variant_note.saved_variant.xpos_start,
                    ref=seqr_variant_note.saved_variant.ref,
                    alt=seqr_variant_note.saved_variant.alt,
                    date_saved=seqr_variant_note.last_modified_date,
                    user=seqr_variant_note.created_by,
                ):
                    print("--- deleting variant note: " + str(new_variant_note))
                    seqr_variant_note.delete()
                    counters['seqr_variant_note_deleted'] += 1

            for indiv in SeqrIndividual.objects.filter(family__project__deprecated_project_id=deprecated_project_id):
                if indiv.guid not in updated_seqr_individual_guids:
                    print("Deleting SeqrIndividual: %s" % indiv)
                    counters["deleted SeqrIndividuals"] += 1
                    indiv.sample_set.all().delete()
                    indiv.delete()

            # delete families that are in SeqrFamily table, but not in BaseProject table
            for f in SeqrFamily.objects.filter(project__deprecated_project_id=deprecated_project_id):
                if f.guid not in updated_seqr_family_guids:
                    print("--- deleting SeqrFamily: %s" % f)
                    counters["deleted SeqrFamilys"] += 1
                    f.delete()

            # if there's a set of samples without individuals
            for sample in SeqrSample.objects.filter(individual__isnull=True):
                print("--- deleting SeqrSample without indiv: %s" % sample)
                counters["deleted SeqrSample"] += 1
                sample.delete()

            # delete projects that are in SeqrProject table, but not in BaseProject table
            #for p in SeqrProject.objects.filter():
            #    if p.guid not in updated_seqr_project_guids:
            #        while True:
            #            i = raw_input('Delete SeqrProject %s? [Y/n]' % p.guid)
            #            if i == 'Y':
            #                p.delete()
            #            else:
            #                print("Keeping %s .." % p.guid)
            #            break

        # delete projects that are in SeqrProject table, but not in BaseProject table
        if not project_ids_to_process:
            all_project_ids = set([project.project_id for project in Project.objects.all()])
            for seqr_project in SeqrProject.objects.all():
                if seqr_project.deprecated_project_id not in all_project_ids:
                    #seqr_project.delete()
                    print("--- Deleting SeqrProject: %s ??" % seqr_project)

        logger.info("Done")
        logger.info("Stats: ")
        for k, v in counters.items():
            if v > 0:
                logger.info("  %s: %s" % (k, v))


def create_sample_records(source_individual, new_project, new_individual, counters):
    loaded_vcf_files = source_individual.vcf_files.filter(dataset_type=VCFFile.DATASET_TYPE_VARIANT_CALLS, loaded_date__isnull=False)
    for loaded_vcf_file in loaded_vcf_files:
        new_sample, sample_created = get_or_create_sample(
            source_individual,
            new_individual,
            sample_type=loaded_vcf_file.sample_type,
            dataset_type=SeqrSample.DATASET_TYPE_VARIANT_CALLS,
            elasticsearch_index=loaded_vcf_file.elasticsearch_index,
            dataset_file_path=loaded_vcf_file.file_path,
            sample_status=SeqrSample.SAMPLE_STATUS_LOADED if loaded_vcf_file else None,
            loaded_date=loaded_vcf_file.loaded_date,
        )

        if sample_created:
            counters['samples_created'] += 1
            logger.info("Created sample: " + str(new_sample.json()))

        if source_individual.bam_file_path:
            new_sample, sample_created = get_or_create_sample(
                source_individual,
                new_individual,
                sample_type=loaded_vcf_file.sample_type,
                dataset_type=SeqrSample.DATASET_TYPE_READ_ALIGNMENTS,
                elasticsearch_index=None,
                dataset_file_path=source_individual.bam_file_path,
                loaded_date=loaded_vcf_file.loaded_date,
                sample_status=SeqrSample.SAMPLE_STATUS_LOADED if loaded_vcf_file else None,
            )

def look_up_vcf_loaded_date(vcf_path):
    vcf_record = get_annotator().get_vcf_file_from_annotator(vcf_path)
    if vcf_record is None:
        raise ValueError("Couldn't find loaded date for %s" % vcf_path)

    loaded_date = vcf_record['_id'].generation_time
    # logger.info("%s data-loaded date: %s" % (vcf_path, loaded_date))
    return loaded_date
<commit_msg>better logging in update samples<commit_after>from bson import json_util
import json
import logging
import pymongo
from tqdm import tqdm
import settings

from django.core.management.base import BaseCommand
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q
from guardian.shortcuts import assign_perm

from seqr.utils.model_sync_utils import convert_html_to_plain_text
from seqr.views.utils.variant_utils import get_or_create_saved_variant
from seqr.views.apis import phenotips_api
from seqr.views.apis.phenotips_api import phenotips_patient_exists, _get_patient_data, _update_individual_phenotips_data
from xbrowse_server.base.models import \
    Project, \
    Family, \
    Individual, \
    VariantNote, \
    ProjectTag, \
    VariantTag, \
    VariantFunctionalData, \
    ProjectCollaborator, VCFFile, AnalysedBy

from seqr.models import \
    Project as SeqrProject, \
    Family as SeqrFamily, \
    Individual as SeqrIndividual, \
    VariantTagType as SeqrVariantTagType, \
    VariantTag as SeqrVariantTag, \
    VariantNote as SeqrVariantNote, \
    VariantFunctionalData as SeqrVariantFunctionalData, \
    Sample as SeqrSample, \
    LocusList, \
    FamilyAnalysedBy as SeqrAnalysedBy, \
    CAN_VIEW, ModelWithGUID

from xbrowse_server.mall import get_datastore, get_annotator

logger = logging.getLogger(__name__)

# switching to python3.6 will make this unnecessary as built-in python dictionaries will be ordered
from collections import OrderedDict, defaultdict
class OrderedDefaultDict(OrderedDict, defaultdict):
    def __init__(self, default_factory=None, *args, **kwargs):
        super(OrderedDefaultDict, self).__init__(*args, **kwargs)
        self.default_factory = default_factory


DEBUG = False   # whether to ask before updating values


class Command(BaseCommand):
    help = 'Transfer projects to the new seqr schema'

    def add_arguments(self, parser):
        parser.add_argument('--reset-all-models', help='This flag causes all records to be cleared from the seqr schema\'s Project, Family, and Individual models before transferring data', action='store_true')
        parser.add_argument('--dont-connect-to-phenotips', help='dont retrieve phenotips internal id and latest data', action='store_true')
        parser.add_argument('project_id', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.')

    def handle(self, *args, **options):
        """transfer project"""
        reset_all_models = options['reset_all_models']
        connect_to_phenotips = not options['dont_connect_to_phenotips']
        project_ids_to_process = options['project_id']
        
        counters = OrderedDefaultDict(int)

        #if reset_all_models:
        #    print("Dropping all records from SeqrProject, SeqrFamily, SeqrIndividual")
        #    SeqrIndividual.objects.all().delete()
        #    SeqrFamily.objects.all().delete()
        #    SeqrProject.objects.all().delete()

        # reset models that'll be regenerated
        #if not project_ids_to_process:
        #    SeqrSample.objects.all().delete()
        #    SeqrVariantTagType.objects.all().delete()
        #    SeqrVariantTag.objects.all().delete()
        #    SeqrVariantNote.objects.all().delete()



        if project_ids_to_process:
            projects = Project.objects.filter(project_id__in=project_ids_to_process)
            logging.info("Processing %s projects" % len(projects))
        else:
            projects = Project.objects.filter(
                ~Q(project_id__contains="DEPRECATED") &
                ~Q(project_name__contains="DEPRECATED") &
                ~Q(project_id__istartswith="temp") &
                ~Q(project_id__istartswith="test_") &
                ~Q(project_name__contains="DISABLED_")
            )
            logging.info("Processing all %s projects" % len(projects))
            project_ids_to_process = [p.project_id for p in projects]


        updated_seqr_project_guids = set()
        updated_seqr_family_guids = set()
        updated_seqr_individual_guids = set()

        for source_project in tqdm(projects, unit=" projects"):
            counters['source_projects'] += 1

            print("Project: " + source_project.project_id)

            # transfer Project data
            new_project, project_created = transfer_project(source_project)
            updated_seqr_project_guids.add(new_project.guid)
            if project_created: counters['projects_created'] += 1

            # transfer Families and Individuals
            source_family_id_to_new_family = {}
            for source_family in Family.objects.filter(project=source_project):
                new_family, family_created = transfer_family(source_family, new_project)

                updated_seqr_family_guids.add(new_family.guid)

                if family_created: counters['families_created'] += 1

                source_family_id_to_new_family[source_family.id] = new_family

                for source_individual in Individual.objects.filter(family=source_family):

                    new_individual, individual_created, phenotips_data_retrieved = transfer_individual(
                        source_individual, new_family, new_project, connect_to_phenotips
                    )

                    updated_seqr_individual_guids.add(new_individual.guid)

                    if individual_created: counters['individuals_created'] += 1
                    if phenotips_data_retrieved: counters['individuals_data_retrieved_from_phenotips'] += 1

                    if source_individual.combined_individuals_info:
                        combined_individuals_info = json.loads(source_individual.combined_individuals_info)
                        """
                        combined_individuals_info json is expected to look like:
                        {
                            'WES' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                            'WGS' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                            'RNA' : {
                                'project_id': from_project.project_id,
                                'family_id': from_f.family_id,
                                'indiv_id': from_i.indiv_id
                            },
                        }
                        """
                        for i, sample_type_i, combined_individuals_info_i in enumerate(combined_individuals_info.items()):
                            source_project_i = Project.objects.get(project_id=combined_individuals_info_i['project_id'])
                            #source_family_i = Project.objects.get(project_id=combined_individuals_info_i['family_id'])
                            source_individual_i = Project.objects.get(project_id=combined_individuals_info_i['indiv_id'])

                            create_sample_records(sample_type_i, source_individual_i, new_project, new_individual, counters)
                    else:
                        create_sample_records(source_individual, new_project, new_individual, counters)
                        #combined_families_info.update({from_project_datatype: {'project_id': from_project.project_id, 'family_id': from_f.family_id}})

                for source_analysed_by in AnalysedBy.objects.filter(family=source_family):
                    new_analysed_by, analysed_by_created = transfer_analysed_by(source_analysed_by, new_family)

                    if analysed_by_created: counters['analysed_by_created'] += 1

            # TODO family groups, cohorts
            for source_variant_tag_type in ProjectTag.objects.filter(project=source_project).order_by('order'):
                new_variant_tag_type, created = get_or_create_variant_tag_type(
                    source_variant_tag_type, new_project)

                for source_variant_tag in VariantTag.objects.filter(project_tag=source_variant_tag_type):
                    new_family = source_family_id_to_new_family.get(source_variant_tag.family.id if source_variant_tag.family else None)
                    new_variant_tag, variant_tag_created = get_or_create_variant_tag(
                        source_variant_tag,
                        new_project,
                        new_family,
                        new_variant_tag_type,
                    )

                    if variant_tag_created: counters['variant_tags_created'] += 1

            for source_variant_functional_data in VariantFunctionalData.objects.filter(family__project=source_project):
                new_family = source_family_id_to_new_family.get(source_variant_functional_data.family.id)

                _, variant_functional_data_created = get_or_create_variant_functional_data(
                    source_variant_functional_data,
                    new_project,
                    new_family,
                )

                if variant_functional_data_created:   counters['variant_functional_data_created'] += 1

            for source_variant_note in VariantNote.objects.filter(project=source_project):
                new_family = source_family_id_to_new_family.get(source_variant_note.family.id if source_variant_note.family else None)

                new_variant_note, variant_note_created = get_or_create_variant_note(
                    source_variant_note,
                    new_project,
                    new_family,
                )

                if variant_note_created:   counters['variant_notes_created'] += 1


        for deprecated_project_id in project_ids_to_process:

            base_project = Project.objects.get(project_id=deprecated_project_id)

            # delete Tag type
            for seqr_variant_tag_type in SeqrVariantTagType.objects.filter(project__deprecated_project_id=deprecated_project_id):
                if not ProjectTag.objects.filter(
                    project=base_project,
                    tag=seqr_variant_tag_type.name,
                    title=seqr_variant_tag_type.description,
                    color=seqr_variant_tag_type.color,
                    order=seqr_variant_tag_type.order
                ):
                    seqr_variant_tag_type.delete()
                    print("--- deleting variant tag type: " + str(seqr_variant_tag_type))
                    counters['seqr_variant_tag_type_deleted'] += 1

            # delete Tag
            delete_count, _ = SeqrVariantTag.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_tag_deleted'] += delete_count
            for seqr_variant_tag in SeqrVariantTag.objects.filter(saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantTag.objects.filter(
                        project_tag__project=base_project,
                        project_tag__tag=seqr_variant_tag.variant_tag_type.name,
                        #project_tag__title=seqr_variant_tag.variant_tag_type.description,
                        xpos=seqr_variant_tag.saved_variant.xpos_start,
                        ref=seqr_variant_tag.saved_variant.ref,
                        alt=seqr_variant_tag.saved_variant.alt,
                ):
                    seqr_variant_tag.delete()
                    print("--- deleting variant tag: " + str(seqr_variant_tag))
                    counters['seqr_variant_tag_deleted'] += 1

            # delete functional data tags
            delete_count, _ = SeqrVariantFunctionalData.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_functional_data_deleted'] += delete_count
            for seqr_variant_functional_data in SeqrVariantFunctionalData.objects.filter(
                    saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantFunctionalData.objects.filter(
                        family__project=base_project,
                        functional_data_tag=seqr_variant_functional_data.functional_data_tag,
                        metadata=seqr_variant_functional_data.metadata,
                        xpos=seqr_variant_functional_data.saved_variant.xpos_start,
                        ref=seqr_variant_functional_data.saved_variant.ref,
                        alt=seqr_variant_functional_data.saved_variant.alt,
                ):
                    seqr_variant_functional_data.delete()
                    print("--- deleting variant tag: " + str(seqr_variant_functional_data))
                    counters['seqr_variant_tag_deleted'] += 1

            # delete Variant Note
            delete_count, _ = SeqrVariantNote.objects.filter(saved_variant__isnull=True).delete()
            counters['seqr_variant_note_deleted'] += delete_count
            for seqr_variant_note in SeqrVariantNote.objects.filter(saved_variant__project__deprecated_project_id=deprecated_project_id):

                if not VariantNote.objects.filter(
                    project=base_project,
                    note=seqr_variant_note.note,
                    xpos=seqr_variant_note.saved_variant.xpos_start,
                    ref=seqr_variant_note.saved_variant.ref,
                    alt=seqr_variant_note.saved_variant.alt,
                    date_saved=seqr_variant_note.last_modified_date,
                    user=seqr_variant_note.created_by,
                ):
                    print("--- deleting variant note: " + str(new_variant_note))
                    seqr_variant_note.delete()
                    counters['seqr_variant_note_deleted'] += 1

            for indiv in SeqrIndividual.objects.filter(family__project__deprecated_project_id=deprecated_project_id):
                if indiv.guid not in updated_seqr_individual_guids:
                    print("Deleting SeqrIndividual: %s" % indiv)
                    counters["deleted SeqrIndividuals"] += 1
                    indiv.sample_set.all().delete()
                    indiv.delete()

            # delete families that are in SeqrFamily table, but not in BaseProject table
            for f in SeqrFamily.objects.filter(project__deprecated_project_id=deprecated_project_id):
                if f.guid not in updated_seqr_family_guids:
                    print("--- deleting SeqrFamily: %s" % f)
                    counters["deleted SeqrFamilys"] += 1
                    f.delete()

            # if there's a set of samples without individuals
            for sample in SeqrSample.objects.filter(individual__isnull=True):
                print("--- deleting SeqrSample without indiv: %s" % sample)
                counters["deleted SeqrSample"] += 1
                sample.delete()

            # delete projects that are in SeqrProject table, but not in BaseProject table
            #for p in SeqrProject.objects.filter():
            #    if p.guid not in updated_seqr_project_guids:
            #        while True:
            #            i = raw_input('Delete SeqrProject %s? [Y/n]' % p.guid)
            #            if i == 'Y':
            #                p.delete()
            #            else:
            #                print("Keeping %s .." % p.guid)
            #            break

        # delete projects that are in SeqrProject table, but not in BaseProject table
        if not project_ids_to_process:
            all_project_ids = set([project.project_id for project in Project.objects.all()])
            for seqr_project in SeqrProject.objects.all():
                if seqr_project.deprecated_project_id not in all_project_ids:
                    #seqr_project.delete()
                    print("--- Deleting SeqrProject: %s ??" % seqr_project)

        logger.info("Done")
        logger.info("Stats: ")
        for k, v in counters.items():
            if v > 0:
                logger.info("  %s: %s" % (k, v))


def create_sample_records(source_individual, new_project, new_individual, counters):
    loaded_vcf_files = source_individual.vcf_files.filter(dataset_type=VCFFile.DATASET_TYPE_VARIANT_CALLS, loaded_date__isnull=False)
    for loaded_vcf_file in loaded_vcf_files:
        new_sample, sample_created = get_or_create_sample(
            source_individual,
            new_individual,
            sample_type=loaded_vcf_file.sample_type,
            dataset_type=SeqrSample.DATASET_TYPE_VARIANT_CALLS,
            elasticsearch_index=loaded_vcf_file.elasticsearch_index,
            dataset_file_path=loaded_vcf_file.file_path,
            sample_status=SeqrSample.SAMPLE_STATUS_LOADED if loaded_vcf_file else None,
            loaded_date=loaded_vcf_file.loaded_date,
        )

        if sample_created:
            counters['samples_created'] += 1
            logger.info("Created sample: " + str(new_sample.json()))

        if source_individual.bam_file_path:
            new_sample, sample_created = get_or_create_sample(
                source_individual,
                new_individual,
                sample_type=loaded_vcf_file.sample_type,
                dataset_type=SeqrSample.DATASET_TYPE_READ_ALIGNMENTS,
                elasticsearch_index=None,
                dataset_file_path=source_individual.bam_file_path,
                loaded_date=loaded_vcf_file.loaded_date,
                sample_status=SeqrSample.SAMPLE_STATUS_LOADED if loaded_vcf_file else None,
            )
            if sample_created:
                counters['samples_created'] += 1
                logger.info("Created alignment sample: " + str(new_sample.json()))


def look_up_vcf_loaded_date(vcf_path):
    vcf_record = get_annotator().get_vcf_file_from_annotator(vcf_path)
    if vcf_record is None:
        raise ValueError("Couldn't find loaded date for %s" % vcf_path)

    loaded_date = vcf_record['_id'].generation_time
    # logger.info("%s data-loaded date: %s" % (vcf_path, loaded_date))
    return loaded_date

{'fraction_non_alphanumeric': 0.07220086829079446, 'fraction_numerical': 0.001133755495948898, 'mean_word_length': 2.187659762009696, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '134434', 'n_tokens_mistral': 10342, 'n_tokens_neox': 9602, 'n_words': 1668}