<commit_before>#!/usr/bin/env python
"""
# Copyright 2013, <NAME>
# Modified Aug 10, 2013
"""


from Bio import SeqIO
from optparse import OptionParser
import sys, os, os.path, time, gzip
from collections import Counter

## Parse options and setup ##
usage = "usage %prog -d [path to directory of raw reads]"
parser = OptionParser(usage=usage)

parser.add_option('-d', '--directory', help="Directory containing read files to de-duplicate",
    action="store", type="str", dest="sample_dir")

(options, args) = parser.parse_args()

if len(args) != 0:
    parser.print_help()
    sys.exit()

sample_dir = options.sample_dir
output_dir = options.sample_dir

#kindly provided by http://stackoverflow.com/questions/7099290/how-to-ignore-hidden-files-using-os-listdir-python
#glob.glob will list hidden files
#this replaces that functionality when hidden files exist, like in my reads from Berkeley
def listdir_nohidden(path):
    for f in os.listdir(path):
        if not f.startswith('.'):
            yield f

def main_sff(infile):
# Set up the global variables
    global count
    global bases
    lcount = 0
    lbases = 0
    lqual = 0

# Open inputs:
    iterator1 = SeqIO.parse(opten(infile, 'r'), "sff")
    try:
        while 1:
            seq1 = iterator1.next()
            count += 1
            lcount += 1
            len = print record.annotations["clip_qual_right"] - print record.annotations["clip_qual_left"]
            bases += len
            lbases += len
            lqual += sum(seq1.letter_annotations['phred_quality'][record.annotations["clip_qual_right"]:print record.annotations["clip_qual_left"]])/len

    except StopIteration:
        pass
    finally:
        print "Finished processing file" +  infile1
        outfile1.write("file\t" + infile1 + "\n")
        outfile1.write("nreads\t" + str(lcount) + "\n")
        outfile1.write("nbases\t" + str(lbases) + "\n")<commit_msg>bugfix<commit_after>#!/usr/bin/env python
"""
# Copyright 2013, <NAME>
# Modified Aug 10, 2013
"""


from Bio import SeqIO
from optparse import OptionParser
import sys, os, os.path, time, gzip
from collections import Counter

## Parse options and setup ##
usage = "usage %prog -d [path to directory of raw reads]"
parser = OptionParser(usage=usage)

parser.add_option('-d', '--directory', help="Directory containing read files to de-duplicate",
    action="store", type="str", dest="sample_dir")

(options, args) = parser.parse_args()

if len(args) != 0:
    parser.print_help()
    sys.exit()

sample_dir = options.sample_dir
output_dir = options.sample_dir

#kindly provided by http://stackoverflow.com/questions/7099290/how-to-ignore-hidden-files-using-os-listdir-python
#glob.glob will list hidden files
#this replaces that functionality when hidden files exist, like in my reads from Berkeley
def listdir_nohidden(path):
    for f in os.listdir(path):
        if not f.startswith('.'):
            yield f

def main_sff(infile):
# Set up the global variables
    global count
    global bases
    lcount = 0
    lbases = 0
    lqual = 0

# Open inputs:
    iterator1 = SeqIO.parse(opten(infile, 'r'), "sff")
    try:
        while 1:
            seq1 = iterator1.next()
            count += 1
            lcount += 1
            len = record.annotations["clip_qual_right"] - print record.annotations["clip_qual_left"]
            bases += len
            lbases += len
            lqual += sum(seq1.letter_annotations['phred_quality'][record.annotations["clip_qual_right"]:print record.annotations["clip_qual_left"]])/len

    except StopIteration:
        pass
    finally:
        print "Finished processing file" +  infile1
        outfile1.write("file\t" + infile1 + "\n")
        outfile1.write("nreads\t" + str(lcount) + "\n")
        outfile1.write("nbases\t" + str(lbases) + "\n")
{'fraction_non_alphanumeric': 0.12424322190050013, 'fraction_numerical': 0.017372992892866545, 'mean_word_length': 3.121475054229935, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5579798', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1184, 'n_words': 322}