<commit_before>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: <EMAIL>
#
import sys
import screed
import argparse


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', nargs='+')
    parser.add_argument('-o', '--output', dest='outfp',
                        help="output file for histogram; defaults to "
                             "<first filename>.errhist in cwd.",
                        type=argparse.FileType('w'), default=None)
    args = parser.parse_args()

    total_bp = 0
    total_seqs = 0

    output = []
    for filename in args.filenames:
        bp = 0
        seqs = 0
        for record in screed.open(filename):
            if seqs % 100000 == 0:
                print >>sys.stderr, '...', filename, seqs
            bp += len(record.sequence)
            seqs += 1

        if seqs == 0:
            print >>sys.stderr, 'No sequences found in %s' % filename
        else:
            avg_len = bp / float(seqs)
            s = '%d bp / %d seqs; %.1f average length -- %s' % (bp,
                                                                seqs,
                                                                avg_len,
                                                                filename)
            print >>sys.stderr, '... found', s
            output.append(s)

            total_bp += bp
            total_seqs += seqs

    if total_seqs == 0:
        print >>args.outfp, \
            'No sequences found in %d files' % len(sys.argv[1:])
    else:
        print >>args.outfp, '---------------'
        print >>args.outfp, "\n".join(output)
        print >>args.outfp, '---------------'
        print >>args.outfp, '%d bp / %d seqs; %.1f average length -- total' % \
            (total_bp, total_seqs, total_bp / float(total_seqs))


if __name__ == '__main__':<commit_msg>a few extra fixes in readstats<commit_after>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: <EMAIL>
#
import sys
import screed
import argparse


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', nargs='+')
    parser.add_argument('-o', '--output', dest='outfp',
                        help="output file for statistics; defaults to stdout.",
                        type=argparse.FileType('w'), default=None)
    args = parser.parse_args()

    total_bp = 0
    total_seqs = 0

    output = []
    for filename in args.filenames:
        bp = 0
        seqs = 0
        for record in screed.open(filename):
            if seqs % 100000 == 0:
                print >>sys.stderr, '...', filename, seqs
            bp += len(record.sequence)
            seqs += 1

        if seqs == 0:
            print >>sys.stderr, 'No sequences found in %s' % filename
        else:
            avg_len = bp / float(seqs)
            s = '%d bp / %d seqs; %.1f average length -- %s' % (bp,
                                                                seqs,
                                                                avg_len,
                                                                filename)
            print >>sys.stderr, '... found', s
            output.append(s)

            total_bp += bp
            total_seqs += seqs

    if total_seqs == 0:
        print >>args.outfp, \
            'No sequences found in %d files' % len(args.filenames)
    else:
        print >>args.outfp, '---------------'
        print >>args.outfp, "\n".join(output)
        print >>args.outfp, '---------------'
        print >>args.outfp, '%d bp / %d seqs; %.1f average length -- total' % \
            (total_bp, total_seqs, total_bp / float(total_seqs))


if __name__ == '__main__':
{'fraction_non_alphanumeric': 0.1360138992305783, 'fraction_numerical': 0.012658227848101266, 'mean_word_length': 1.4102870813397128, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4783194', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1121, 'n_words': 325}