<commit_before>package io.compgen.ngsutils.cli.fastq;

import io.compgen.cmdline.annotation.Command;
import io.compgen.cmdline.annotation.Exec;
import io.compgen.cmdline.annotation.Option;
import io.compgen.cmdline.annotation.UnnamedArg;
import io.compgen.cmdline.exceptions.CommandArgumentException;
import io.compgen.cmdline.impl.AbstractOutputCommand;
import io.compgen.ngsutils.fastq.Fastq;
import io.compgen.ngsutils.fastq.FastqRead;
import io.compgen.ngsutils.fastq.FastqReader;
import io.compgen.ngsutils.fastq.filter.BlacklistFilter;
import io.compgen.ngsutils.fastq.filter.FastqFilter;
import io.compgen.ngsutils.fastq.filter.FilteringException;
import io.compgen.ngsutils.fastq.filter.PairedFilter;
import io.compgen.ngsutils.fastq.filter.PrefixQualFilter;
import io.compgen.ngsutils.fastq.filter.PrefixTrimFilter;
import io.compgen.ngsutils.fastq.filter.SeqTrimFilter;
import io.compgen.ngsutils.fastq.filter.SizeFilter;
import io.compgen.ngsutils.fastq.filter.SuffixQualFilter;
import io.compgen.ngsutils.fastq.filter.SuffixTrimFilter;
import io.compgen.ngsutils.fastq.filter.WhitelistFilter;
import io.compgen.ngsutils.fastq.filter.WildcardFilter;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;

@Command(name = "fastq-filter", desc = "Filters reads from a FASTQ file.", category="fastq")
public class FastqFilterCli extends AbstractOutputCommand {
    private boolean paired = false;
    private int suffixQuality = -1;
    private int prefixQuality = -1;
    private int prefixTrimLength = -1;
    private int suffixTrimLength = -1;
    private int minimumSize = -1;
    private int maxWildcard = -1;
    
    private String trimSeq = null;
    private String trimSeq1 = null;
    private String trimSeq2 = null;
    private int trimMinOverlap = 4;
    private double trimMinPctMatch = 0.9;
    
    private String whitelist = null;
    private String blacklist = null;

    private String filename;
    private String summaryFilename=null;
    
    public FastqFilterCli() {
    }

    @UnnamedArg(name = "FILE", defaultValue="-")
    public void setFilename(String filename) throws CommandArgumentException {
        if (filename == null) {
            throw new CommandArgumentException("Missing file!");
        }
        if (!filename.equals("-")) {
            if (!new File(filename).exists()) {
                throw new CommandArgumentException("Missing file: "+filename);
            }
        }
        this.filename = filename;
    }

    @Option(desc="Write summary of filters to file", name="summary", helpValue="fname")
    public void setSummaryFilename(String summaryFilename) {
        this.summaryFilename = summaryFilename;
    }

    @Option(desc="Sequence trim filter (adapters, all reads) (Default: not used)", name="trim-seq")
    public void setTrimSeq(String trimSeq) {
        this.trimSeq = trimSeq;
    }

    @Option(desc="Sequence trim filter (adapters, read 1) (Default: not used)", name="trim-seq1")
    public void setTrimSeq1(String trimSeq1) {
        this.trimSeq1 = trimSeq1;
    }

    @Option(desc="Sequence trim filter (adapters, read 2) (Default: not used)", name="trim-seq2")
    public void setTrimSeq2(String trimSeq2) {
        this.trimSeq2 = trimSeq2;
    }

    @Option(desc="Sequence trim minimum overlap", name="trim-overlap", defaultValue="6")
    public void setTrimMinOverlap(int trimMinOverlap) {
        this.trimMinOverlap = trimMinOverlap;
    }

    @Option(desc="Sequence trim minimum percent match", name="trim-pct", defaultValue="0.9")
    public void setTrimMinPctMatch(double trimMinPctMatch) {
        this.trimMinPctMatch = trimMinPctMatch;
    }
    
    @Option(desc="Paired filter (for interleaved files) (default: not used)", name="paired")
    public void setPaired(boolean paired) {
        this.paired = paired;
    }

    @Option(desc="Suffix quality filter (minimum quality)", name="suffixqual", defaultValue="3")
    public void setSuffixQualityFilter(int suffixQuality) {
        this.suffixQuality = suffixQuality;
    }

    @Option(desc="Prefix quality filter (minimum quality)", name="prefixqual", defaultValue="3")
    public void setPrefixQualityFilter(int prefixQuality) {
        this.prefixQuality = prefixQuality;
    }

    @Option(desc="Prefix fixed-base trim length", name="prefixtrim", defaultValue = "0")
    public void setPrefixTrimLength(int prefixTrimLength) {
        this.prefixTrimLength = prefixTrimLength;
    }

    @Option(desc="Suffix fixed-base trim length", name="suffixtrim", defaultValue = "0")
    public void setSufffixTrimLength(int suffixTrimLength) {
        this.suffixTrimLength = suffixTrimLength;
    }

    @Option(desc="Minimum read length", name="size", defaultValue = "35")
    public void setMinimumSize(int minimumSize) {
        this.minimumSize = minimumSize;
    }

    @Option(desc="Maximum wildcard calls (-1 to disable)", name="wildcard", defaultValue = "2")
    public void setMaxWildcard(int maxWildcard) {
        this.maxWildcard = maxWildcard;
    }

    @Option(desc="Blacklist filename (read names)", name="blacklist")
    public void setBlacklist(String blacklist) {
        this.blacklist = blacklist;
    }

    @Option(desc="Whitelist filename (read names)", name="whitelist")
    public void setWhitelist(String whitelist) {
        this.whitelist = whitelist;
    }
    
    @Exec
    public void exec() throws IOException, CommandArgumentException, FilteringException {
        if (whitelist != null && blacklist != null) {
            throw new CommandArgumentException("You can not specify both a whitelist and a blacklist!");
        }
        
        FastqReader reader = Fastq.open(filename);

        if (verbose) {
            System.err.println("Filtering file:" + filename);
        }
<commit_msg>Change default to 50bp size<commit_after>package io.compgen.ngsutils.cli.fastq;

import io.compgen.cmdline.annotation.Command;
import io.compgen.cmdline.annotation.Exec;
import io.compgen.cmdline.annotation.Option;
import io.compgen.cmdline.annotation.UnnamedArg;
import io.compgen.cmdline.exceptions.CommandArgumentException;
import io.compgen.cmdline.impl.AbstractOutputCommand;
import io.compgen.ngsutils.fastq.Fastq;
import io.compgen.ngsutils.fastq.FastqRead;
import io.compgen.ngsutils.fastq.FastqReader;
import io.compgen.ngsutils.fastq.filter.BlacklistFilter;
import io.compgen.ngsutils.fastq.filter.FastqFilter;
import io.compgen.ngsutils.fastq.filter.FilteringException;
import io.compgen.ngsutils.fastq.filter.PairedFilter;
import io.compgen.ngsutils.fastq.filter.PrefixQualFilter;
import io.compgen.ngsutils.fastq.filter.PrefixTrimFilter;
import io.compgen.ngsutils.fastq.filter.SeqTrimFilter;
import io.compgen.ngsutils.fastq.filter.SizeFilter;
import io.compgen.ngsutils.fastq.filter.SuffixQualFilter;
import io.compgen.ngsutils.fastq.filter.SuffixTrimFilter;
import io.compgen.ngsutils.fastq.filter.WhitelistFilter;
import io.compgen.ngsutils.fastq.filter.WildcardFilter;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;

@Command(name = "fastq-filter", desc = "Filters reads from a FASTQ file.", category="fastq")
public class FastqFilterCli extends AbstractOutputCommand {
    private boolean paired = false;
    private int suffixQuality = -1;
    private int prefixQuality = -1;
    private int prefixTrimLength = -1;
    private int suffixTrimLength = -1;
    private int minimumSize = -1;
    private int maxWildcard = -1;
    
    private String trimSeq = null;
    private String trimSeq1 = null;
    private String trimSeq2 = null;
    private int trimMinOverlap = 4;
    private double trimMinPctMatch = 0.9;
    
    private String whitelist = null;
    private String blacklist = null;

    private String filename;
    private String summaryFilename=null;
    
    public FastqFilterCli() {
    }

    @UnnamedArg(name = "FILE", defaultValue="-")
    public void setFilename(String filename) throws CommandArgumentException {
        if (filename == null) {
            throw new CommandArgumentException("Missing file!");
        }
        if (!filename.equals("-")) {
            if (!new File(filename).exists()) {
                throw new CommandArgumentException("Missing file: "+filename);
            }
        }
        this.filename = filename;
    }

    @Option(desc="Write summary of filters to file", name="summary", helpValue="fname")
    public void setSummaryFilename(String summaryFilename) {
        this.summaryFilename = summaryFilename;
    }

    @Option(desc="Sequence trim filter (adapters, all reads) (Default: not used)", name="trim-seq")
    public void setTrimSeq(String trimSeq) {
        this.trimSeq = trimSeq;
    }

    @Option(desc="Sequence trim filter (adapters, read 1) (Default: not used)", name="trim-seq1")
    public void setTrimSeq1(String trimSeq1) {
        this.trimSeq1 = trimSeq1;
    }

    @Option(desc="Sequence trim filter (adapters, read 2) (Default: not used)", name="trim-seq2")
    public void setTrimSeq2(String trimSeq2) {
        this.trimSeq2 = trimSeq2;
    }

    @Option(desc="Sequence trim minimum overlap", name="trim-overlap", defaultValue="6")
    public void setTrimMinOverlap(int trimMinOverlap) {
        this.trimMinOverlap = trimMinOverlap;
    }

    @Option(desc="Sequence trim minimum percent match", name="trim-pct", defaultValue="0.9")
    public void setTrimMinPctMatch(double trimMinPctMatch) {
        this.trimMinPctMatch = trimMinPctMatch;
    }
    
    @Option(desc="Paired filter (for interleaved files) (default: not used)", name="paired")
    public void setPaired(boolean paired) {
        this.paired = paired;
    }

    @Option(desc="Suffix quality filter (minimum quality)", name="suffixqual", defaultValue="3")
    public void setSuffixQualityFilter(int suffixQuality) {
        this.suffixQuality = suffixQuality;
    }

    @Option(desc="Prefix quality filter (minimum quality)", name="prefixqual", defaultValue="3")
    public void setPrefixQualityFilter(int prefixQuality) {
        this.prefixQuality = prefixQuality;
    }

    @Option(desc="Prefix fixed-base trim length", name="prefixtrim", defaultValue = "0")
    public void setPrefixTrimLength(int prefixTrimLength) {
        this.prefixTrimLength = prefixTrimLength;
    }

    @Option(desc="Suffix fixed-base trim length", name="suffixtrim", defaultValue = "0")
    public void setSufffixTrimLength(int suffixTrimLength) {
        this.suffixTrimLength = suffixTrimLength;
    }

    @Option(desc="Minimum read length", name="size", defaultValue = "50")
    public void setMinimumSize(int minimumSize) {
        this.minimumSize = minimumSize;
    }

    @Option(desc="Maximum wildcard calls (-1 to disable)", name="wildcard", defaultValue = "2")
    public void setMaxWildcard(int maxWildcard) {
        this.maxWildcard = maxWildcard;
    }

    @Option(desc="Blacklist filename (read names)", name="blacklist")
    public void setBlacklist(String blacklist) {
        this.blacklist = blacklist;
    }

    @Option(desc="Whitelist filename (read names)", name="whitelist")
    public void setWhitelist(String whitelist) {
        this.whitelist = whitelist;
    }
    
    @Exec
    public void exec() throws IOException, CommandArgumentException, FilteringException {
        if (whitelist != null && blacklist != null) {
            throw new CommandArgumentException("You can not specify both a whitelist and a blacklist!");
        }
        
        FastqReader reader = Fastq.open(filename);

        if (verbose) {
            System.err.println("Filtering file:" + filename);
        }

{'fraction_non_alphanumeric': 0.10752505520638696, 'fraction_numerical': 0.005945303210463734, 'mean_word_length': 3.7270172621437174, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4041502', 'n_tokens_mistral': 3584, 'n_tokens_neox': 3411, 'n_words': 802}