<commit_before>#!/usr/bin/env python
"""
# Copyright 2013, <NAME>, <NAME>, <NAME>
# Modified Aug 9, 2013
"""


from Bio import SeqIO
from optparse import OptionParser
import sys, os, os.path, time, gzip
from collections import Counter

## Parse options and setup ##
usage = "usage %prog -d [path to directory of raw reads] -o [output file prefix (path + name)]"
usage += "\n\te.g. %prog read1.fastq read2.fastq ./out/output_base"
parser = OptionParser(usage=usage)

parser.add_option('-d', '--directory', help="Directory containing read files to de-duplicate",
                    action="store", type="str", dest="sample_dir")

parser.add_option('-o', '--output', help="Directory to output de-duplicated reads",
                    action="store", type="str", dest="output_dir")


(options, args) = parser.parse_args()

sample_dir = options.sample_dir
output_dir = options.output_dir

if len(args) != 0 or sample_dir is None or output_dir is None:
    parser.print_help()
    sys.exit()


#kindly provided by http://stackoverflow.com/questions/7099290/how-to-ignore-hidden-files-using-os-listdir-python
#glob.glob will list hidden files
#this replaces that functionality when hidden files exist, like in my reads from Berkeley
def listdir_nohidden(path):
    for f in sorted(os.listdir(path), key=str.lower):
        if not f.startswith('.'):
            yield f


def main(infile1, infile2, outfile1, outfile2):
#Set up the global variables
    global count
    global i
    global duplicates
    global rev<commit_msg>more descriptive help<commit_after>#!/usr/bin/env python
"""
# Copyright 2013, <NAME>, <NAME>, <NAME>
# Modified Aug 9, 2013
"""


from Bio import SeqIO
from optparse import OptionParser
import sys, os, os.path, time, gzip
from collections import Counter

## Parse options and setup ##
usage = "usage %prog -d [path to directory of raw reads] -o [output file prefix (path + name)]"
usage += "\n\te.g. %prog read1.fastq read2.fastq ./out/output_base"
parser = OptionParser(usage=usage)

parser.add_option('-d', '--directory', help="Directory containing read files to de-duplicate",
                    action="store", type="str", dest="sample_dir")

parser.add_option('-o', '--output', help="Directory + prefix to output de-duplicated reads",
                    action="store", type="str", dest="output_dir")


(options, args) = parser.parse_args()

sample_dir = options.sample_dir
output_dir = options.output_dir

if len(args) != 0 or sample_dir is None or output_dir is None:
    parser.print_help()
    sys.exit()


#kindly provided by http://stackoverflow.com/questions/7099290/how-to-ignore-hidden-files-using-os-listdir-python
#glob.glob will list hidden files
#this replaces that functionality when hidden files exist, like in my reads from Berkeley
def listdir_nohidden(path):
    for f in sorted(os.listdir(path), key=str.lower):
        if not f.startswith('.'):
            yield f


def main(infile1, infile2, outfile1, outfile2):
#Set up the global variables
    global count
    global i
    global duplicates
    global rev
{'fraction_non_alphanumeric': 0.12455050670153645, 'fraction_numerical': 0.015037593984962405, 'mean_word_length': 3.8341232227488153, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4788131', 'n_tokens_mistral': 989, 'n_tokens_neox': 928, 'n_words': 304}