<commit_before>#!/usr/bin/env perl

#input: fasta
#outputs:  ${out_prefix}.aa${pid}.faa, ${out_prefix}.aa${pid}.mapping
#            OR
#          ${out_prefix}.rna${pid}.fna or ${out_prefix}.rna${pid}.mapping
#            OR
#          ${out_prefix}.dna${pid}.fna or ${out_prefix}.dna${pid}.mapping

use strict;
use warnings;
no warnings('once');

use PipelineAWE;
use Getopt::Long;
use Cwd;
umask 000;

# options
my $out_prefix = "cluster";
my $fasta   = "";
my $pid     = 90;
my $mem     = 16;
my $aa      = 0;
my $rna     = 0;
my $dna     = 0;
my $help    = 0;
my $options = GetOptions (
        "out_prefix=s" => \$out_prefix,
        "input=s" => \$fasta,
		"pid=i"   => \$pid,
		"mem=i"   => \$mem,
		"aa!"     => \$aa,
        "rna!"    => \$rna,
        "dna!"    => \$dna,
        "help!"   => \$help
);

if ($help){
    print get_usage();
    exit 0;
}elsif (length($fasta)==0){
    print STDERR "ERROR: An input file was not specified.\n";
    print STDERR get_usage();
    exit __LINE__;
}elsif (! -e $fasta){
    print STDERR "ERROR: The input sequence file [$fasta] does not exist.\n";
    print STDERR get_usage();
    exit __LINE__;
}elsif ((! $pid) || ($pid < 40)){
    print STDERR "ERROR: The percent identity must be greater than 50\n";
    print STDERR get_usage();
    exit __LINE__;
}

my ($cmd, $word, $code, $output);
if ($aa) {
    ($cmd, $word, $code, $output) = ("cd-hit", word_length("aa", $pid), "aa", $out_prefix.".aa".$pid.".faa");
} else {
    $code = $rna ? 'rna' : 'dna';
    ($cmd, $word, $output) = ("cd-hit-est", word_length($code, $pid), $out_prefix.".".$code.$pid.".fna");
}
my $memory = $mem * 1024;

print "$cmd -n $word -d 0 -T 0 -M $memory -c 0.$pid -i $fasta -o $output\n";
PipelineAWE::run_cmd("$cmd -n $word -d 0 -T 0 -M $memory -c 0.$pid -i $fasta -o $output");

# turn $output.clstr into $out_prefix.".".$code.$pid.".mapping"
open(IN, "<".$output.".clstr") || exit __LINE__;
open(OUT, ">".$out_prefix.".".$code.$pid.".mapping") || exit __LINE__;
my $clust = [];
my $parse = "";
my $junk  = <IN>; # kill first line

while (my $line = <IN>) {
    chomp $line;
    # process previous cluster
    if ($line =~ /^>Cluster/) {
        $parse = parse_clust($clust);
        if ($parse) {
            print OUT $parse;
        }
        $clust = [];
    } else {
        push @$clust, $line;
    }
}
# process last cluster
$parse = parse_clust($clust);
if ($parse) {
    print OUT $parse;
}
close(IN);
close(OUT);<commit_msg>cleanup<commit_after>#!/usr/bin/env perl

#input: fasta
#outputs:  ${out_prefix}.aa${pid}.faa, ${out_prefix}.aa${pid}.mapping
#            OR
#          ${out_prefix}.rna${pid}.fna or ${out_prefix}.rna${pid}.mapping
#            OR
#          ${out_prefix}.dna${pid}.fna or ${out_prefix}.dna${pid}.mapping

use strict;
use warnings;
no warnings('once');

use PipelineAWE;
use Getopt::Long;
use Cwd;
umask 000;

# options
my $out_prefix = "cluster";
my $fasta   = "";
my $pid     = 90;
my $mem     = 16;
my $aa      = 0;
my $rna     = 0;
my $dna     = 0;
my $help    = 0;
my $options = GetOptions (
        "out_prefix=s" => \$out_prefix,
        "input=s" => \$fasta,
		"pid=i"   => \$pid,
		"mem=i"   => \$mem,
		"aa!"     => \$aa,
        "rna!"    => \$rna,
        "dna!"    => \$dna,
        "help!"   => \$help
);

if ($help){
    print get_usage();
    exit 0;
}elsif (length($fasta)==0){
    print STDERR "ERROR: An input file was not specified.\n";
    print STDERR get_usage();
    exit __LINE__;
}elsif (! -e $fasta){
    print STDERR "ERROR: The input sequence file [$fasta] does not exist.\n";
    print STDERR get_usage();
    exit __LINE__;
}elsif ((! $pid) || ($pid < 40)){
    print STDERR "ERROR: The percent identity must be greater than 50\n";
    print STDERR get_usage();
    exit __LINE__;
}

my ($cmd, $word, $code, $output);
if ($aa) {
    ($cmd, $word, $code, $output) = ("cd-hit", word_length("aa", $pid), "aa", $out_prefix.".aa".$pid.".faa");
} else {
    $code = $rna ? 'rna' : 'dna';
    ($cmd, $word, $output) = ("cd-hit-est", word_length($code, $pid), $out_prefix.".".$code.$pid.".fna");
}
my $memory = $mem * 1024;

print "$cmd -n $word -d 0 -T 0 -M $memory -c 0.$pid -i $fasta -o $output\n";
PipelineAWE::run_cmd("$cmd -n $word -d 0 -T 0 -M $memory -c 0.$pid -i $fasta -o $output");

# turn $output.clstr into $out_prefix.".".$code.$pid.".mapping"
my $clust = [];
my $parse = "";
open(IN, "<".$output.".clstr") || exit __LINE__;
open(OUT, ">".$out_prefix.".".$code.$pid.".mapping") || exit __LINE__;

while (my $line = <IN>) {
    chomp $line;
    # process previous cluster
    if ($line =~ /^>Cluster/) {
        $parse = parse_clust($clust);
        if ($parse) {
            print OUT $parse;
        }
        $clust = [];
    } else {
        push @$clust, $line;
    }
}
# process last cluster
$parse = parse_clust($clust);
if ($parse) {
    print OUT $parse;
}
close(IN);
close(OUT);
{'fraction_non_alphanumeric': 0.214227309893704, 'fraction_numerical': 0.011038430089942763, 'mean_word_length': 2.6679160419790104, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6499640', 'n_tokens_mistral': 1979, 'n_tokens_neox': 1900, 'n_words': 501}