<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8 tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
# Copyright © <NAME> <<EMAIL>>
#
# Distributed under terms of the MIT license
"""
====================================================================================

          FILE: vcf_simplify (python 3)
        AUTHOR: <NAME>, <EMAIL>
  ORGANIZATION: Stanford University
       LICENSE: MIT License, Property of Stanford, Use however you wish
       VERSION: 0.3
       CREATED: 2014-01-21 17:38
 Last modified: 2014-01-22 19:56

   DESCRIPTION: Take a compressed vcf file such as
                ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/ALL.chr1.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz
                and create a simplified matrix where all genotypes are represented
                as 0/1/2 where 0: homozygote_1; 1:heterozygote; 2: homozygote_2.

                Out file format (tab delimited):
                CHR(e.g.1/MT)\\tPOS(e.g.10583)\\tSNP_ID(e.g.rs58108140)\\tref\\talt\\tqual\\tfilter\\t[sample_1]\\t[sample_2]\\t...\\t[sample_n]

                Execution time on a single 1000genomes file is
                2647.92s user 9.00s system 97% cpu 45:16.76 total

  REQUIREMENTS: 1. A 1000genomes-style VCF file with GT:DS:GL style genotypes (see note)
                2. A 1000genomes-style panel file
                    ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/integrated_call_samples.20101123.ALL.panel

          NOTE: The vcf files must be gzipped, and the genotypes must be encoded as
                0|0, 0|1, 1|1

 USAGE EXAMPLE: ./vcf_simplify.py -p integrated_call_samples.20101123.ALL.panel\
                ALL.chr1.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz\
                ALL.chr2.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz

====================================================================================
"""
import sys, re
from multiprocessing import Pool

# Default threads
default_threads = 8

def vcf_simplify(vcf_file, logfile=sys.stderr, verbose=False, outfile=''):
    """Take a 1000genomes style vcf file (MUST BE GZIPPED) and
       simplify it to:

       chr\\tpos\\trsID\\tref\\talt\\tqual\\tfilter\\tsample_1\\t[sample_2]\\t...[sample_n]\\n"""<commit_msg>Documentation update<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8 tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
# Copyright © <NAME> <<EMAIL>>
#
# Distributed under terms of the MIT license
"""
================================================================================================

          FILE: vcf_simplify (python 3) (multithreading)
        AUTHOR: <NAME>, <EMAIL>
  ORGANIZATION: Stanford University
       LICENSE: MIT License, Property of Stanford, Use however you wish
       VERSION: 0.3
       CREATED: 2014-01-21 17:38
 Last modified: 2014-01-22 20:07

   DESCRIPTION: Take a compressed vcf file such as
                ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/ALL.chr1.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz
                and create a simplified matrix where all genotypes are represented
                as 0/1/2 where 0: homozygote_1; 1:heterozygote; 2: homozygote_2.

                Out file format (tab delimited):
                CHR(e.g.1/MT)\\tPOS(e.g.10583)\\tSNP_ID(e.g.rs58108140)\\tref\\talt\\tqual\\tfilter\\t[sample_1]\\t[sample_2]\\t...\\t[sample_n]

                Execution time on a single 1000genomes file is
                2647.92s user 9.00s system 97% cpu 45:16.76 total

                Additionally, it is possible to filter a 1000genomes style vcf file, or 
                a previously simplified vcf file by population, region, or platform.
                Filtering requires a 1000genomes style panel file, such as:
                ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/integrated_call_samples.20101123.ALL.panel

          NOTE: The vcf files must be gzipped, and the genotypes must be encoded as
                0|0, 0|1, 1|0, 1|1

USAGE EXAMPLES: Simply vcf files:
                ./vcf_simplify.py ALL.chr1.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz\\
                ALL.chr2.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz

                Filter 1000genomes vcf files by population:
                ./vcf_simplify.py -p integrated_call_samples.20101123.ALL.panel\\
                --filter_population CEU,YRI \\
                ALL.chr1.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz\\
                ALL.chr2.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz

=============================================================================================
"""
import sys, re
from multiprocessing import Pool

# Default threads
default_threads = 8

def vcf_simplify(vcf_file, logfile=sys.stderr, verbose=False, outfile=''):
    """Take a 1000genomes style vcf file (MUST BE GZIPPED) and
       simplify it to:

       chr\\tpos\\trsID\\tref\\talt\\tqual\\tfilter\\tsample_1\\t[sample_2]\\t...[sample_n]\\n"""
{'fraction_non_alphanumeric': 0.16663492668063226, 'fraction_numerical': 0.0666539706722529, 'mean_word_length': 2.8731563421828907, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3776908', 'n_tokens_mistral': 2002, 'n_tokens_neox': 1638, 'n_words': 370}