<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: 10.10.2015
#@author: <NAME>
#@contact: <EMAIL>

import sys
import argparse
import re
from collections import defaultdict

def get_revcomp(sequence):
    '''Return reverse complementary sequence.

    >>> complementary('AT CG')
    'CGAT'

    '''
    c = dict(zip('ATCGNatcgn[]', 'TAGCNtagcn]['))
    return ''.join(c.get(nucleotide, '') for nucleotide in reversed(sequence))


def sc_iter_fasta_brute(file_name):
    """ Iter over fasta file."""
    header = None
    seq = []
    with open(file_name) as fh:
        data = fh.readlines()
        for line in data:
            if line.startswith(">"):
                if seq:
                    sequence = "".join(seq)
                    yield (header, sequence)
                header = line
                seq = []
                continue
            seq.append(line)
        if seq or header:
            sequence = "".join(seq)
            yield (header, sequence)


if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='Fasta file to kmers file.')
	parser.add_argument('-i','--input', help='Fasta input', required=True)
	parser.add_argument('-o','--output', help='Kmer output', required=True)
	parser.add_argument('-k','--k', help='K lengths', required=True)
	args = vars(parser.parse_args())
	fasta_file = args["input"]
	output = args["output"]
	k = int(args["k"])

	kmers = defaultdict(int)
	for header, sequence in sc_iter_fasta_brute(fasta_file):
		sequence = sequence.upper()
		sequence = re.sub("\s+", "", sequence, re.S|re.M)
		for i in xrange(0, len(sequence)-k+1):
			kmer = sequence[i:i+k]
			kmers[kmer] += 1
			rkmer = get_revcomp(kmer)
			kmers[rkmer] += 1
	with open(output, "w") as fh:
		for kmer in kmers:
			fh.write("%s\t%s\n" % (kmer, kmers[kmer]))

<commit_msg>make_library script refactored<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# created: 10.10.2015
# author: <NAME>
# contact: <EMAIL>

import argparse
import re
from collections import defaultdict


def get_revcomp(seq):
    """Return reverse complementary sequence.

    >>> get_revcomp('AT CG')
    'CGAT'

    """
    c = dict(zip('ATCGNatcgn[]', 'TAGCNtagcn]['))
    return ''.join(c.get(nucleotide, '') for nucleotide in reversed(
        seq))


def sc_iter_fasta_brute(file_name):
    """ Iter over fasta file."""
    seq_header = None
    seq = []
    with open(file_name) as file_handler:
        data = file_handler.readlines()
        for line in data:
            if line.startswith(">"):
                if seq:
                    yield (seq_header, "".join(seq))
                seq_header = line
                seq = []
                continue
            seq.append(line)
        if seq or seq_header:
            yield (seq_header, "".join(seq))


if __name__ == '__main__':

    parser = argparse.ArgumentParser(
        description='Create a filf of kmers from a FASTA file')
    parser.add_argument('-i', '--input', help='FASTA input',
                        required=True)
    parser.add_argument('-o', '--output', help='kmer output',
                        required=True)
    parser.add_argument('-k', '--k', help='length of kmers',
                        required=True)
    args = vars(parser.parse_args())
    fasta_file = args["input"]
    output = args["output"]
    k = int(args["k"])

    kmers = defaultdict(int)
    for header, sequence in sc_iter_fasta_brute(fasta_file):
        sequence = sequence.upper()
        sequence = re.sub("\s+", "", sequence, re.S | re.M)
        for i in xrange(0, len(sequence)-k+1):
            kmer = sequence[i:i+k]
            kmers[kmer] += 1
            rkmer = get_revcomp(kmer)
            kmers[rkmer] += 1
    with open(output, "w") as fh:
        for kmer in kmers:
            fh.write("%s\t%s\n" % (kmer, kmers[kmer]))
{'fraction_non_alphanumeric': 0.15164892235782915, 'fraction_numerical': 0.006751493118670475, 'mean_word_length': 2.3968253968253967, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7107815', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1254, 'n_words': 301}