<commit_before>#!/usr/bin/env node

/**
 */

var opts = require('commander');
var fasta = require('bionode-fasta');
var fs = require('fs');
var through = require('through2')


// Helper function to verbose option
function increaseVerbosity(v, total) {
  return total + 1;
}

// Override handling of unknown options - someone decided exit code 2 was better than the default 1
opts.unknownOption = function(flag) {
    console.error();
    console.error("  error: unknown option `%s'", flag);
    console.error();
    process.exit(2);
}

// Option parsing
opts
  .version('1.0.0')
  .usage('[options] contigs.fasta [contigs2.fasta ...]')
  .description('Print fasta stats')
  .option('-m, --minlen <n>', 'Minimum length sequence to include in stats (default=0)', parseInt, 0)
  .option('-v, --verbose', "Print more stuff about what's happening", increaseVerbosity, 0)
  .parse(process.argv);

// Default to reading stdin if there are no files specified
if (opts.args == 0)
    opts.args = ["/dev/stdin"];



////////////////////////////////////////////////////////////

// Stream to process a fasta file for stats
function process_fasta(file) {
    var bp=0;
    var n=0;
    var min,max;
    var stream = through.obj(next_seq, done)
    return stream

    // Filter, and collect stats on each sequence
    function next_seq(data, enc, next) {
        var l = data.seq.length
        if (l>=opts.minlen) {
            if (opts.verbose>=2)
                console.error([file, data.id, l].join("\t"))
            min = n==0 ? l : Math.min(l,min)
            max = n==0 ? l : Math.max(l,max)
            n += 1
            bp += l
        }
        next()
    }

    function done() {
        this.push([n, bp, min, Math.floor(bp/n), max]);
        this.push(null);
    }
}

// Loop over all the files specified
function process_files(files) {
    // Done when we have no files left!
    if (files.length==0)
        return
    var file = files.shift()

    if (opts.verbose>=1)
        console.error("Processing: "+file);

    // Read the file, pipe through the fasta parser, then through our filter, then output as appropriate
    fs.createReadStream(file)<commit_msg>Handle files with no sequences passing filter<commit_after>#!/usr/bin/env node

/**
 */

var opts = require('commander');
var fasta = require('bionode-fasta');
var fs = require('fs');
var through = require('through2')


// Helper function to verbose option
function increaseVerbosity(v, total) {
  return total + 1;
}

// Override handling of unknown options - someone decided exit code 2 was better than the default 1
opts.unknownOption = function(flag) {
    console.error();
    console.error("  error: unknown option `%s'", flag);
    console.error();
    process.exit(2);
}

// Option parsing
opts
  .version('1.0.0')
  .usage('[options] contigs.fasta [contigs2.fasta ...]')
  .description('Print fasta stats')
  .option('-m, --minlen <n>', 'Minimum length sequence to include in stats (default=0)', parseInt, 0)
  .option('-v, --verbose', "Print more stuff about what's happening", increaseVerbosity, 0)
  .parse(process.argv);

// Default to reading stdin if there are no files specified
if (opts.args == 0)
    opts.args = ["/dev/stdin"];



////////////////////////////////////////////////////////////

// Stream to process a fasta file for stats
function process_fasta(file) {
    var bp=0;
    var n=0;
    var min,max;
    var stream = through.obj(next_seq, done)
    return stream

    // Filter, and collect stats on each sequence
    function next_seq(data, enc, next) {
        var l = data.seq.length
        if (l>=opts.minlen) {
            if (opts.verbose>=2)
                console.error([file, data.id, l].join("\t"))
            min = n==0 ? l : Math.min(l,min)
            max = n==0 ? l : Math.max(l,max)
            n += 1
            bp += l
        }
        next()
    }

    function done() {
        if (n>0)
            this.push([n, bp, min, Math.floor(bp/n), max]);
        else
            this.push([n, bp, '-', '-', '-']);
        this.push(null);
    }
}

// Loop over all the files specified
function process_files(files) {
    // Done when we have no files left!
    if (files.length==0)
        return
    var file = files.shift()

    if (opts.verbose>=1)
        console.error("Processing: "+file);

    // Read the file, pipe through the fasta parser, then through our filter, then output as appropriate
    fs.createReadStream(file)
{'fraction_non_alphanumeric': 0.16692913385826771, 'fraction_numerical': 0.00967379077615298, 'mean_word_length': 2.3303370786516853, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7168507', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1356, 'n_words': 424}