<commit_before>import os
import glob
import shutil
import tarfile
import platform

from django.core.management.base import BaseCommand, CommandError
from django.conf import settings

from lib.utils.io import get_url_content
from lib.utils import clogging
log = clogging.getColorLogger(__name__)


class Command(BaseCommand):
    help = "Setup go-vcf-tools"

    def handle(self, *args, **options):
        tmp_dir = os.path.join(settings.BASE_DIR, 'tmp')
        bin_dir = os.path.join(settings.BASE_DIR, 'bin')

        log.info('Fetching go-vcf-tools ...')
        url = '{repo}/releases/download/{tag}/{os_platform}.amd64.tar.gz'.format(repo='https://github.com/knmkr/go-vcf-tools',
                                                                                 tag='0.0.1',  # TODO: get tag from command argument
                                                                                 os_platform=platform.system().lower())
        tar_gz = os.path.join(tmp_dir, 'go-vcf-tools.tar.gz')
        get_url_content(url, tar_gz)

        log.info('Extracting go-vcf-tools ...')
        with tarfile.open(tar_gz, 'r') as tar:
            dst = os.path.join(tmp_dir, 'go-vcf-tools')
            tar.extractall(dst)
            for tool in glob.glob(os.path.join(tmp_dir, 'go-vcf-tools', '*', 'vcf-*')):
                shutil.copy(tool, bin_dir)

        os.remove(tar_gz)
        shutil.rmtree(dst)

        log.info('Fetching RsMergeArch ...')
        url = 'http://ftp.ncbi.nih.gov/snp/organisms/human_9606_b144_GRCh37p13/database/organism_data/RsMergeArch.bcp.gz'
        get_url_content(url, settings.RS_MERGE_ARCH_PATH, if_not_exists=True)

        log.info('Done.')<commit_msg>update go-vcf-tools version<commit_after>import os
import glob
import shutil
import tarfile
import platform

from django.core.management.base import BaseCommand, CommandError
from django.conf import settings

from lib.utils.io import get_url_content
from lib.utils import clogging
log = clogging.getColorLogger(__name__)


class Command(BaseCommand):
    help = "Setup go-vcf-tools"

    def handle(self, *args, **options):
        tmp_dir = os.path.join(settings.BASE_DIR, 'tmp')
        bin_dir = os.path.join(settings.BASE_DIR, 'bin')

        log.info('Fetching go-vcf-tools ...')
        url = '{repo}/releases/download/{tag}/{os_platform}.amd64.tar.gz'.format(repo='https://github.com/knmkr/go-vcf-tools',
                                                                                 tag='0.0.2',
                                                                                 os_platform=platform.system().lower())
        tar_gz = os.path.join(tmp_dir, 'go-vcf-tools.tar.gz')
        get_url_content(url, tar_gz)

        log.info('Extracting go-vcf-tools ...')
        with tarfile.open(tar_gz, 'r') as tar:
            dst = os.path.join(tmp_dir, 'go-vcf-tools')
            tar.extractall(dst)
            for tool in glob.glob(os.path.join(tmp_dir, 'go-vcf-tools', '*', 'vcf-*')):
                shutil.copy(tool, bin_dir)

        os.remove(tar_gz)
        shutil.rmtree(dst)

        log.info('Fetching RsMergeArch ...')
        url = 'http://ftp.ncbi.nih.gov/snp/organisms/human_9606_b144_GRCh37p13/database/organism_data/RsMergeArch.bcp.gz'
        get_url_content(url, settings.RS_MERGE_ARCH_PATH, if_not_exists=True)

        log.info('Done.')
{'fraction_non_alphanumeric': 0.13047363717605004, 'fraction_numerical': 0.009532320524277629, 'mean_word_length': 2.147141518275539, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '980147', 'n_tokens_mistral': 1101, 'n_tokens_neox': 1049, 'n_words': 175}