<commit_before>import os
import csv
import gzip
from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db import models, transaction
from django.conf import settings
from django.utils import timezone
from django.utils.dateparse import parse_date

from apps.snp.models import Snp, get_freqs
from apps.gwascatalog.models import GwasCatalogSnp, GwasCatalogPhenotype
from cleanup.population import get_population
from cleanup.odds_ratio_or_beta_coeff import get_odds_ratio_or_beta_coeff, get_ci_and_unit
from cleanup.errors import GwasCatalogParseError
from cleanup.reliability_rank import get_reliability_rank
from cleanup.risk_allele import get_database_strand_allele, AMBIGOUS
from lib.utils.io import get_url_content
from lib.utils.pg import text2pg_array
from lib.utils import clogging
log = clogging.getColorLogger(__name__)


class Command(BaseCommand):
    help = "Update GWAS Catalog"

    def handle(self, *args, **options):
        current_tz = timezone.get_current_timezone()

        if not os.path.exists(settings.GWASCATALOG_DIR):
            os.makedirs(settings.GWASCATALOG_DIR)

        # TODO: automatically choose latest version
        log.info('Fetching latest gwascatalog...')
        catalog_path = os.path.join(settings.GWASCATALOG_DIR, 'dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-cleaned.tsv.gz')
        get_url_content(url='https://github.com/knmkr/dbsnp-pg-min/releases/download/0.5.2/dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-cleaned.tsv.gz',
                        dst=catalog_path,
                        if_not_exists=True)

        log.info('Fetching latest gwascatalog allele freq...')
        catalog_freq_path = os.path.join(settings.GWASCATALOG_DIR, 'dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-snps-allele-freq.tsv.gz')
        get_url_content(url='https://github.com/knmkr/dbsnp-pg-min/releases/download/0.5.2/dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-snps-allele-freq.tsv.gz',
                        dst=catalog_freq_path,
                        if_not_exists=True)

        # - Gwas Catalog Allele Freq
        log.info('Updating snp allele freq records for gwascatalog...')
        num_created = 0
        num_updated = 0

        with transaction.atomic():
            for record in csv.DictReader(gzip.open(catalog_freq_path, 'rb'), delimiter='\t',
                                         fieldnames=['snp_id_current', 'allele', 'freq', 'populations']):
                snp, created = Snp.objects.update_or_create(
                    snp_id_current=record['snp_id_current'],
                    defaults={'allele': text2pg_array(record['allele']),
                              'freq': text2pg_array(record['freq']),
                              'population': record['populations']}
                )
                if created:
                    num_created += 1
                else:
                    num_updated += 1

        log.info('updated: {} records'.format(num_updated))
        log.info('created: {} records'.format(num_created))

        # - Gwas Catalog<commit_msg>fix population of allele freq records<commit_after>import os
import csv
import gzip
from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db import models, transaction
from django.conf import settings
from django.utils import timezone
from django.utils.dateparse import parse_date

from apps.snp.models import Snp, get_freqs
from apps.gwascatalog.models import GwasCatalogSnp, GwasCatalogPhenotype
from cleanup.population import get_population
from cleanup.odds_ratio_or_beta_coeff import get_odds_ratio_or_beta_coeff, get_ci_and_unit
from cleanup.errors import GwasCatalogParseError
from cleanup.reliability_rank import get_reliability_rank
from cleanup.risk_allele import get_database_strand_allele, AMBIGOUS
from lib.utils.io import get_url_content
from lib.utils.pg import text2pg_array
from lib.utils import clogging
log = clogging.getColorLogger(__name__)


class Command(BaseCommand):
    help = "Update GWAS Catalog"

    def handle(self, *args, **options):
        current_tz = timezone.get_current_timezone()

        if not os.path.exists(settings.GWASCATALOG_DIR):
            os.makedirs(settings.GWASCATALOG_DIR)

        # TODO: automatically choose latest version
        log.info('Fetching latest gwascatalog...')
        catalog_path = os.path.join(settings.GWASCATALOG_DIR, 'dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-cleaned.tsv.gz')
        get_url_content(url='https://github.com/knmkr/dbsnp-pg-min/releases/download/0.5.2/dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-cleaned.tsv.gz',
                        dst=catalog_path,
                        if_not_exists=True)

        log.info('Fetching latest gwascatalog allele freq...')
        catalog_freq_path = os.path.join(settings.GWASCATALOG_DIR, 'dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-snps-allele-freq.tsv.gz')
        get_url_content(url='https://github.com/knmkr/dbsnp-pg-min/releases/download/0.5.2/dbsnp-pg-min-0.5.2-b144-GRCh37.gwascatalog-snps-allele-freq.tsv.gz',
                        dst=catalog_freq_path,
                        if_not_exists=True)

        # - Gwas Catalog Allele Freq
        log.info('Updating snp allele freq records for gwascatalog...')
        num_created = 0
        num_updated = 0

        with transaction.atomic():
            for record in csv.DictReader(gzip.open(catalog_freq_path, 'rb'), delimiter='\t',
                                         fieldnames=['snp_id_current', 'allele', 'freq', 'populations']):
                snp, created = Snp.objects.update_or_create(
                    snp_id_current=record['snp_id_current'],
                    population=record['populations'],
                    defaults={'allele': text2pg_array(record['allele']),
                              'freq': text2pg_array(record['freq'])}
                )
                if created:
                    num_created += 1
                else:
                    num_updated += 1

        log.info('updated: {} records'.format(num_updated))
        log.info('created: {} records'.format(num_created))

        # - Gwas Catalog
{'fraction_non_alphanumeric': 0.10027494743651949, 'fraction_numerical': 0.01455604075691412, 'mean_word_length': 2.6919402985074625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5217821', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1887, 'n_words': 341}