<commit_before>import subprocess
import tempfile

transtab = str.maketrans('ACGTNacgtn','TGCANtgcan')

revcompdict = {}
def revcomp(sequence):
    """Reverse complement a string

    :param sequence: The DNA string (all caps)

    This function includes a caching mechanism, so watch memory usage!
    """
    if(sequence in revcompdict):
        return revcompdict[sequence]
    tmp=sequence[::-1].translate(transtab)
    revcompdict[sequence]=tmp
    return(tmp)


def fileOpen(fname,mode='r'):
    """Open a file, including gzip files

    :param fname: The filename to open.  Gzip files are distinguished by ending in '.gz'
    :param mode: File mode for opening [default 'r']
    :returns: An open file handle

    Note: the gzip module in python is REALLY slow, so this function uses
    subprocess and command-line gzip instead.
    """
    # gzip in python is REALLY slow, so use pipes instead.
    if(fname.endswith('.gz')):
        if(mode.startswith('r')):
            return subprocess.Popen(['gunzip -c %s' % fname],stdout=subprocess.PIPE,shell=True).stdout
        if(mode.startswith('w')):
            return subprocess.Popen(['gzip > %s' % fname],stdin=subprocess.PIPE,shell=True).stdin
    else:
        return open(fname,'r')


def sortVcfBySequence(vcf,seqnames,seqmap=None):
    """Sort a tabixed VCF file by sequence names

    >>> import vcf
    >>> v = vcf.Reader('vcf.gz')
    >>> from seqtools.utils import sortVcfBySequence
    >>> w = vcf.Writer(open('sorted.vcf','w'),v)
    >>> faifile = open('ucsc.hg19.fasta.fai')
    >>> seqnames = [x.split('\t')[0] for x in faifile]
    >>> for rec in sortVcfBySequence(v,seqnames):
        w.write_record(rec)
    >>> w.close()
    >>> v.close()"""

    if(seqmap):
        revseqmap = dict([(v,k) for (k,v) in list(seqmap.items())])

    print(seqmap)
    
    for s in seqnames:
        # seqmap maps seqnames to tabix index names
        if(seqmap is not None):
            region = vcf.fetch(seqmap[s],start=0)
        else:
            try:
                region = vcf.fetch(s,start=0)
                print(region)
            except ValueError:
                print("here",s)<commit_msg>Using python3 gzip for incredible performance improvements over subprocess gzip<commit_after>import subprocess
import tempfile
import gzip

transtab = str.maketrans('ACGTNacgtn','TGCANtgcan')

revcompdict = {}
def revcomp(sequence):
    """Reverse complement a string

    :param sequence: The DNA string (all caps)

    This function includes a caching mechanism, so watch memory usage!
    """
    if(sequence in revcompdict):
        return revcompdict[sequence]
    tmp=sequence[::-1].translate(transtab)
    revcompdict[sequence]=tmp
    return(tmp)


def fileOpen(fname,mode='rt',encoding='latin-1'):
    """Open a file, including gzip files

    :param fname: The filename to open.  Gzip files are distinguished by ending in '.gz'
    :param mode: File mode for opening [default 'r']
    :returns: An open file handle

    Note: the gzip module in python is REALLY slow, so this function uses
    subprocess and command-line gzip instead.
    """
    # gzip in python is REALLY slow, so use pipes instead.
    if(fname.endswith('.gz')):
        return gzip.open(fname,mode=mode,encoding=encoding)
    else:
        return open(fname,mode)


def sortVcfBySequence(vcf,seqnames,seqmap=None):
    """Sort a tabixed VCF file by sequence names

    >>> import vcf
    >>> v = vcf.Reader('vcf.gz')
    >>> from seqtools.utils import sortVcfBySequence
    >>> w = vcf.Writer(open('sorted.vcf','w'),v)
    >>> faifile = open('ucsc.hg19.fasta.fai')
    >>> seqnames = [x.split('\t')[0] for x in faifile]
    >>> for rec in sortVcfBySequence(v,seqnames):
        w.write_record(rec)
    >>> w.close()
    >>> v.close()"""

    if(seqmap):
        revseqmap = dict([(v,k) for (k,v) in list(seqmap.items())])

    print(seqmap)
    
    for s in seqnames:
        # seqmap maps seqnames to tabix index names
        if(seqmap is not None):
            region = vcf.fetch(seqmap[s],start=0)
        else:
            try:
                region = vcf.fetch(s,start=0)
                print(region)
            except ValueError:
                print("here",s)
{'fraction_non_alphanumeric': 0.12802275960170698, 'fraction_numerical': 0.003319108582266477, 'mean_word_length': 2.8529680365296803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5663520', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1341, 'n_words': 369}