<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
load_mongo.py

Load script for the mongo db.
Should take a directory as input, like the load part of vcf.py, and populate the mongo database.

Created by <NAME> on 2014-11-10.
Copyright (c) 2014 __MoonsoInc__. All rights reserved.

"""

# Questions for Mats:
# - Should we first check if variant is in database and then update?
# - Is it more effective to, like in sql, update the database with alot of data at the same time or can we do one and one?
# - How to handle general and specific?
# - How to initiate an index on a field?
# - What does this pymongo.ASCENDING means?



from __future__ import (absolute_import, unicode_literals, print_function,
                        division)

import sys
import os

import io
import json
import click



from datetime import datetime
from pymongo import (ASCENDING, DESCENDING)
from mongoengine import connect, DoesNotExist
from mongoengine.connection import get_db


from .config_parser import ConfigParser
from .utils import (get_case, get_institute, get_mongo_variant)
from ...models import Institute


from vcf_parser import VCFParser

from pprint import pprint as pp

import scout


def load_mongo_db(scout_configs, config_file=None, family_type='cmms',
                  mongo_db='variantDatabase', variant_type='clinical',
                  username=None, password=<PASSWORD>, port=27017,
                  rank_score_treshold = 0, host='localhost',verbose = False):
  """Populate a moongo database with information from ped and variant files."""
  # get root path of the Flask app
  # project_root = '/'.join(app.root_path.split('/')[0:-1])

  ####### Check if the vcf file is on the proper format #######
  vcf_file = scout_configs['load_vcf']
  splitted_vcf_file_name = os.path.splitext(vcf_file)
  vcf_ending = splitted_vcf_file_name[-1]
  if vcf_ending != '.vcf':
    if vcf_ending == '.gz':
      vcf_ending = os.path.splitext(splitted_vcf_file_name)[-1]
      if vcf_ending != '.vcf':
        print("Please use the correct prefix of your vcf file('.vcf/.vcf.gz')",
               file=sys.stderr)
        sys.exit(0)
    else:
      if vcf_ending != '.vcf':
        print("Please use the correct prefix of your vcf file('.vcf/.vcf.gz')",
                file=sys.stderr)
        sys.exit(0)
  
  connect(mongo_db, host=host, port=port, username=username,
          password=password)
  
  variant_database = get_db()
  
  ped_file = scout_configs['ped']
  
  if verbose:
    print("\nvcf_file:\t%s\nped_file:\t%s\nconfig_file:\t%s\nfamily_type:\t%s\n"
          "mongo_db:\t%s\ninstitutes:\t%s\n" % (vcf_file, ped_file, 
          config_file, family_type, mongo_db, ','.join(scout_configs['institutes'])),
          file=sys.stderr)

  ######## Parse the config file to check for keys ########
  config_object = ConfigParser(config_file)

  ######## Add the institute to the mongo db: ########

  # institutes is a list with institute objects
  institutes = []
  for institute_name in scout_configs['institutes']:
    institutes.append(get_institute(institute_name))

  # If the institute exists we work on the old one
  for i, institute in enumerate(institutes):
    try:
      if Institute.objects.get(internal_id = institute.internal_id):
        institutes[i] = Institute.objects.get(internal_id = institute.internal_id)
    except DoesNotExist:
      if verbose:
        print('New institute!', file=sys.stderr)
  
  
  ######## Get the cases and add them to the mongo db: ########
  
  case = get_case(scout_configs, family_type)
  
  if verbose:
    print('Case found in %s: %s' % (ped_file, case.display_name),
          file=sys.stderr)
  
  # Add the case to its institute(s)
  for institute_object in institutes:
    if case not in institute_object.cases:
      institute_object.cases.append(case)
  
    institute_object.save()
  
  if variant_type == 'research':
    case['is_research'] = True
  
  else:
    case.save()
  
  ######## Get the variants and add them to the mongo db: ########
  
  variant_parser = VCFParser(infile=vcf_file, split_variants=True)
  nr_of_variants = 0
  start_inserting_variants = datetime.now()
  
  # Get the individuals to see which we should include in the analysis
  ped_individuals = []
  for individual in case.individuals:
    ped_individuals.append(individual.individual_id)
  
  # Check which individuals that exists in the vcf file:
  individuals = []
  for individual in ped_individuals:
    if individual in variant_parser.individuals:
      individuals.append(individual)
    else:
      if verbose:
        print("Individual %s is present in ped file but not in vcf!\n"
              "Continuing analysis..." % individual, file=sys.stderr)

  if verbose:
    print('Start parsing variants...', file=sys.stderr)

  ########## If a rank score treshold is used check if it is below that treshold ##########
  for variant in variant_parser:
    if not float(variant['rank_scores'][case.display_name]) > rank_score_treshold:
      break<commit_msg>Fixed problem with existing cases<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
load_mongo.py

Load script for the mongo db.
Should take a directory as input, like the load part of vcf.py, and populate the mongo database.

Created by <NAME> on 2014-11-10.
Copyright (c) 2014 __MoonsoInc__. All rights reserved.

"""

# Questions for Mats:
# - Should we first check if variant is in database and then update?
# - Is it more effective to, like in sql, update the database with alot of data at the same time or can we do one and one?
# - How to handle general and specific?
# - How to initiate an index on a field?
# - What does this pymongo.ASCENDING means?



from __future__ import (absolute_import, unicode_literals, print_function,
                        division)

import sys
import os

import io
import json
import click



from datetime import datetime
from pymongo import (ASCENDING, DESCENDING)
from mongoengine import connect, DoesNotExist
from mongoengine.connection import get_db


from .config_parser import ConfigParser
from .utils import (get_case, get_institute, get_mongo_variant)
from ...models import Institute


from vcf_parser import VCFParser

from pprint import pprint as pp

import scout


def load_mongo_db(scout_configs, config_file=None, family_type='cmms',
                  mongo_db='variantDatabase', variant_type='clinical',
                  username=None, password=<PASSWORD>, port=27017,
                  rank_score_treshold = 0, host='localhost',verbose = False):
  """Populate a moongo database with information from ped and variant files."""
  # get root path of the Flask app
  # project_root = '/'.join(app.root_path.split('/')[0:-1])

  ####### Check if the vcf file is on the proper format #######
  vcf_file = scout_configs['load_vcf']
  splitted_vcf_file_name = os.path.splitext(vcf_file)
  vcf_ending = splitted_vcf_file_name[-1]
  if vcf_ending != '.vcf':
    if vcf_ending == '.gz':
      vcf_ending = os.path.splitext(splitted_vcf_file_name)[-1]
      if vcf_ending != '.vcf':
        print("Please use the correct prefix of your vcf file('.vcf/.vcf.gz')",
               file=sys.stderr)
        sys.exit(0)
    else:
      if vcf_ending != '.vcf':
        print("Please use the correct prefix of your vcf file('.vcf/.vcf.gz')",
                file=sys.stderr)
        sys.exit(0)
  
  connect(mongo_db, host=host, port=port, username=username,
          password=password)
  
  variant_database = get_db()
  
  ped_file = scout_configs['ped']
  
  if verbose:
    print("\nvcf_file:\t%s\nped_file:\t%s\nconfig_file:\t%s\nfamily_type:\t%s\n"
          "mongo_db:\t%s\ninstitutes:\t%s\n" % (vcf_file, ped_file, 
          config_file, family_type, mongo_db, ','.join(scout_configs['institutes'])),
          file=sys.stderr)

  ######## Parse the config file to check for keys ########
  config_object = ConfigParser(config_file)

  ######## Add the institute to the mongo db: ########

  # institutes is a list with institute objects
  institutes = []
  for institute_name in scout_configs['institutes']:
    institutes.append(get_institute(institute_name))

  # If the institute exists we work on the old one
  for i, institute in enumerate(institutes):
    try:
      if Institute.objects.get(internal_id = institute.internal_id):
        institutes[i] = Institute.objects.get(internal_id = institute.internal_id)
    except DoesNotExist:
      if verbose:
        print('New institute!', file=sys.stderr)
  
  
  ######## Get the cases and add them to the mongo db: ########
  
  case = get_case(scout_configs, family_type)
  
  if verbose:
    print('Case found in %s: %s' % (ped_file, case.display_name),
          file=sys.stderr)

  # Add the case to its institute(s)
  for institute_object in institutes:
    if case not in institute_object.cases:
      institute_object.cases.append(case)
  
    institute_object.save()
  
  try:
    existing_case = Case.objects.get(case_id = case.case_id)
    if variant_type=='research':
      existing_case.research_gene_lists = case.research_gene_lists
      existing_case.is_research = True
    else:
      existing_case.clinical_gene_lists = case.clinical_gene_lists
    existing_case.save()
  except DoesNotExist:
    if verbose:
      print('New case!', file=sys.stderr)
    case.save()
    
  ######## Get the variants and add them to the mongo db: ########
  
  variant_parser = VCFParser(infile=vcf_file, split_variants=True)
  nr_of_variants = 0
  start_inserting_variants = datetime.now()
  
  # Get the individuals to see which we should include in the analysis
  ped_individuals = []
  for individual in case.individuals:
    ped_individuals.append(individual.individual_id)
  
  # Check which individuals that exists in the vcf file:
  individuals = []
  for individual in ped_individuals:
    if individual in variant_parser.individuals:
      individuals.append(individual)
    else:
      if verbose:
        print("Individual %s is present in ped file but not in vcf!\n"
              "Continuing analysis..." % individual, file=sys.stderr)

  if verbose:
    print('Start parsing variants...', file=sys.stderr)

  ########## If a rank score treshold is used check if it is below that treshold ##########
  for variant in variant_parser:
    if not float(variant['rank_scores'][case.display_name]) > rank_score_treshold:
      break
{'fraction_non_alphanumeric': 0.1093012048192771, 'fraction_numerical': 0.0050120481927710845, 'mean_word_length': 3.4078164825828376, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '904389', 'n_tokens_mistral': 3409, 'n_tokens_neox': 3207, 'n_words': 966}