<commit_before># -*- coding: utf-8 -*-

import sys, os
import re
import glob
import datetime
import subprocess
from optparse import make_option
from termcolor import colored
from pymongo import MongoClient
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from django.conf import settings

from lib.mysql.import_bioq import import_bioq
from lib.mongo.import_refFlat import import_refFlat
from lib.mongo.import_OMIM import OMIMParser
from lib.mongo.clean_catalog import clean_catalog
from lib.mongo.import_catalog import import_catalog
from lib.mongo.import_population_pca import import_population_pca
from lib.mongo.import_genomes import import_genomes

from lib.common import clean_file_name
from lib.mongo.import_variants import import_variants
from lib.utils.date import today_date, today_str
from lib.utils import clogging
log = clogging.getColorLogger(__name__)
from utils.io import get_url_content


def date2datetime(d):
    return datetime.datetime.combine(d, datetime.time())


class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option(
            "--gwascatalog",
            action="store_true",
            dest="gwascatalog",
            help=colored("Import GWAS Catalog into database", "green")
        ),
        make_option(
            "--demodata",
            action="store_true",
            dest="demodata",
            help=colored("Import Demo data & Demo user (if not exists) into database", "green")
        ),
        make_option(
            "--bioq",
            action="store_true",
            dest="bioq",
            help=colored("Import BioQ into database", "green")
        ),
        make_option(
            "--pdb",
            action="store_true",
            dest="pdb",
            help=colored("Import PDB", "green")
        ),
        make_option(
            "--refflat",
            action="store_true",
            dest="refflat",
            help=colored("Import refFlat into database", "green")
        ),
        make_option(
            "--omim",
            action="store_true",
            dest="omim",
            help=colored("Import OMIM into database", "green")
        ),
        make_option(
            "--population_pca",
            action="store_true",
            dest="population_pca",
            help=colored("Import Population PCA data into database", "green")
        ),
        make_option(
            "--genomes",
            action="store_true",
            dest="genomes",
            help=colored("Import genome data (to be used as cron job)", "green")
        ),

    )

    def handle(self, *args, **options):
        if options["gwascatalog"]:
            log.info('Try to import latest gwascatalog...')

            # check if gwascatalog.<today>.txt is exists
            latest_catalog = os.path.join('data', 'gwascatalog.' + today_str + '.txt')
            latest_date = today_date

            if os.path.exists(latest_catalog):
                log.info('Latest gwascatalog exists.')
            else:
                # get latest gwascatalog from official web site
                log.info('Getting latest gwascatalog form official web site...')
                log.info('Getting from {0} ...'.format(settings.GWASCATALOG_URL))

                try:
                    get_url_content(url=settings.GWASCATALOG_URL, dst=latest_catalog)
                    # download_file(url=settings.GWASCATALOG_URL, dst=latest_catalog)
                except IOError:
                    log.error('Could not get latest gwascatalog.')
                    log.error('Proceed importing gwascatalog with local gwascatalog? (may not latest)')
                    yn = raw_input('y/n > ')

                    if yn == 'y':
                        # get latest in local
                        re_datetime = re.compile('gwascatalog\.(\d+)_(\d+)_(\d+)\.txt')
                        local_latest_date = datetime.date(2000, 01, 01)
                        for c in glob.glob(os.path.join(settings.BASE_DIR, 'data', 'gwascatalog.*.txt')):
                            tmp_date = re_datetime.findall(c)
                            if tmp_date:
                                tmp_date = datetime.date(int(tmp_date[0][0]), int(tmp_date[0][1]), int(tmp_date[0][2]))
                                if tmp_date > local_latest_date:
                                    local_latest_date = tmp_date

                        latest_catalog = os.path.join('data', 'gwascatalog.' + str(local_latest_date).replace('-', '_') + '.txt')
                        latest_date = local_latest_date

                        log.warn('=======================================================')
                        log.warn('Use local latest gwascatalog: {0}'.format(latest_catalog))
                        log.warn('=======================================================')

                    else:
                        log.info('Importing gwascatalog stopped.')
                        sys.exit()

            latest_catalog_cleaned = latest_catalog.replace('.txt', '.cleaned.txt')

            if os.path.exists(latest_catalog_cleaned):
                log.info('Latest gwascatalog.cleaned exists.')
            else:
                # do `clean_catalog`
                log.info('Cleaning latest gwascatalog...')
                clean_catalog(latest_catalog, latest_catalog_cleaned)
<commit_msg>add settings.BASE_DIR to dst path of gwascatalog<commit_after># -*- coding: utf-8 -*-

import sys, os
import re
import glob
import datetime
import subprocess
from optparse import make_option
from termcolor import colored
from pymongo import MongoClient
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from django.conf import settings

from lib.mysql.import_bioq import import_bioq
from lib.mongo.import_refFlat import import_refFlat
from lib.mongo.import_OMIM import OMIMParser
from lib.mongo.clean_catalog import clean_catalog
from lib.mongo.import_catalog import import_catalog
from lib.mongo.import_population_pca import import_population_pca
from lib.mongo.import_genomes import import_genomes

from lib.common import clean_file_name
from lib.mongo.import_variants import import_variants
from lib.utils.date import today_date, today_str
from lib.utils import clogging
log = clogging.getColorLogger(__name__)
from utils.io import get_url_content


def date2datetime(d):
    return datetime.datetime.combine(d, datetime.time())


class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option(
            "--gwascatalog",
            action="store_true",
            dest="gwascatalog",
            help=colored("Import GWAS Catalog into database", "green")
        ),
        make_option(
            "--demodata",
            action="store_true",
            dest="demodata",
            help=colored("Import Demo data & Demo user (if not exists) into database", "green")
        ),
        make_option(
            "--bioq",
            action="store_true",
            dest="bioq",
            help=colored("Import BioQ into database", "green")
        ),
        make_option(
            "--pdb",
            action="store_true",
            dest="pdb",
            help=colored("Import PDB", "green")
        ),
        make_option(
            "--refflat",
            action="store_true",
            dest="refflat",
            help=colored("Import refFlat into database", "green")
        ),
        make_option(
            "--omim",
            action="store_true",
            dest="omim",
            help=colored("Import OMIM into database", "green")
        ),
        make_option(
            "--population_pca",
            action="store_true",
            dest="population_pca",
            help=colored("Import Population PCA data into database", "green")
        ),
        make_option(
            "--genomes",
            action="store_true",
            dest="genomes",
            help=colored("Import genome data (to be used as cron job)", "green")
        ),

    )

    def handle(self, *args, **options):
        if options["gwascatalog"]:
            log.info('Try to import latest gwascatalog...')

            # check if gwascatalog.<today>.txt is exists
            latest_catalog = os.path.join(settings.BASE_DIR, 'data', 'gwascatalog.' + today_str + '.txt')
            latest_date = today_date

            if os.path.exists(latest_catalog):
                log.info('Latest gwascatalog exists.')
            else:
                # get latest gwascatalog from official web site
                log.info('Getting latest gwascatalog form official web site...')
                log.info('Getting from {0} ...'.format(settings.GWASCATALOG_URL))

                try:
                    get_url_content(url=settings.GWASCATALOG_URL, dst=latest_catalog)
                    # download_file(url=settings.GWASCATALOG_URL, dst=latest_catalog)
                except IOError:
                    log.error('Could not get latest gwascatalog.')
                    log.error('Proceed importing gwascatalog with local gwascatalog? (may not latest)')
                    yn = raw_input('y/n > ')

                    if yn == 'y':
                        # get latest in local
                        re_datetime = re.compile('gwascatalog\.(\d+)_(\d+)_(\d+)\.txt')
                        local_latest_date = datetime.date(2000, 01, 01)
                        for c in glob.glob(os.path.join(settings.BASE_DIR, 'data', 'gwascatalog.*.txt')):
                            tmp_date = re_datetime.findall(c)
                            if tmp_date:
                                tmp_date = datetime.date(int(tmp_date[0][0]), int(tmp_date[0][1]), int(tmp_date[0][2]))
                                if tmp_date > local_latest_date:
                                    local_latest_date = tmp_date

                        latest_catalog = os.path.join(settings.BASE_DIR, 'data', 'gwascatalog.' + str(local_latest_date).replace('-', '_') + '.txt')
                        latest_date = local_latest_date

                        log.warn('=======================================================')
                        log.warn('Use local latest gwascatalog: {0}'.format(latest_catalog))
                        log.warn('=======================================================')

                    else:
                        log.info('Importing gwascatalog stopped.')
                        sys.exit()

            latest_catalog_cleaned = latest_catalog.replace('.txt', '.cleaned.txt')

            if os.path.exists(latest_catalog_cleaned):
                log.info('Latest gwascatalog.cleaned exists.')
            else:
                # do `clean_catalog`
                log.info('Cleaning latest gwascatalog...')
                clean_catalog(latest_catalog, latest_catalog_cleaned)

{'fraction_non_alphanumeric': 0.12265617854202872, 'fraction_numerical': 0.0032927833165645294, 'mean_word_length': 1.9157333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7383159', 'n_tokens_mistral': 3026, 'n_tokens_neox': 2824, 'n_words': 646}