<commit_before>package Tephra::LTR::LTRRefine;

use 5.014;
use Moose;
use File::Spec;
use File::Find;
use File::Basename;
use File::Copy          qw(move);
use Sort::Naturally     qw(nsort);
use List::UtilsBy       qw(nsort_by);
use List::Util          qw(sum max);
use Cwd                 qw(abs_path);
use Bio::GFF3::LowLevel qw(gff3_parse_feature gff3_format_feature);
use Bio::DB::HTS::Kseq;
use Bio::DB::HTS::Faidx;
use Set::IntervalTree;
use Path::Class::File;
use Carp 'croak';
#use Data::Dump::Color;
use namespace::autoclean;

with 'Tephra::Role::Util', 
     'Tephra::Role::Logger';

=head1 NAME

Tephra::LTR::LTRRefine - Refine LTR predictions based on multiple criteria

=head1 VERSION

Version 0.12.2

=cut

our $VERSION = '0.12.2';
$VERSION = eval $VERSION;

has genome => (
      is       => 'ro',
      isa      => 'Path::Class::File',
      required => 1,
      coerce   => 1,
);

has outfile => (
      is        => 'ro',
      isa       => 'Str',
      predicate => 'has_outfile',
);

has logfile => (
      is        => 'ro',
      isa       => 'Str',
      predicate => 'has_logfile',
);

has n_threshold => (
      is       => 'ro',
      isa      => 'Num',
      required => 0,
      default  => 0.30,
);

has remove_dup_domains => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_remove_dup_domains',
    lazy      => 1,
    default   => 0,
);

has remove_tnp_domains => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_remove_tnp_domains',
    lazy      => 1,
    default   => 0,
);

has domains_required => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_domains_required',
    lazy      => 1,
    default   => 0,
);

has is_trim => (
    is        => 'ro',
    isa       => 'Bool',
    lazy      => 1,
    default   => 0,
);

#
# methods
#
sub collect_features {
    my $self = shift;
    my ($gff_thresh_ref) = @_;
    my ($gff, $pid_thresh) = @{$gff_thresh_ref}{qw(gff pid_threshold)};

    my (%features, %intervals, %coord_map);
    my ($source, $seq_id, $start, $end, $length, $region);

    open my $gffio, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";

    while (my $line = <$gffio>) {
	chomp $line;
	next if $line =~ /^#/;
	my $feature = gff3_parse_feature( $line );
	if ($feature->{type} eq 'repeat_region') {
	    $region = @{$feature->{attributes}{ID}}[0];
	    ($seq_id, $start, $end)  = @{$feature}{qw(seq_id start end)};
	    $length = $end - $start + 1; 
	    my $region_key = join "||", $region."_".$pid_thresh, $start, $end, $length;
	    $features{$seq_id}{$region_key} = [];
	    #$coord_map{$region."_".$pid_thresh} = join "||", $seq_id, $start, $end;
	    $intervals{$region."_".$pid_thresh} = join "||", $start, $end, $length;
	}
	
	#if ($feature->{type} ne 'repeat_region') {
	if ($feature->{type} =~ /repeat_region|target_site_duplication/) {
	    my $id = @{$feature->{attributes}{ID}}[0];
	    my $parent = @{$feature->{attributes}{Parent}}[0];
	        
	    if ($feature->{start} >= $start && $feature->{end} <= $end) {
		my $region_key = join "||", $region."_".$pid_thresh, $start, $end, $length;
		if ($parent =~ /(repeat_region\d+)/) {
		    my $old_parent = $1;
		    my $new_parent = join "_", $old_parent, $pid_thresh;
		    $parent =~ s/$old_parent/$new_parent/;
		    $feature->{attributes}{Parent}[0] = $parent;
		}
		push @{$features{$seq_id}{$region_key}}, $feature;
	    }
	}
    }

    my ($filtered, $stats) = $self->filter_compound_elements(\%features);
    return ({ collected_features => $filtered, stats => $stats, intervals => \%intervals}); #, region_map => \%coord_map });
}

sub filter_compound_elements {
    my $self = shift;
    my ($features) = @_;
    my (@pdoms, %classII);<commit_msg>be sure to capture all subfeatures of repeat region<commit_after>package Tephra::LTR::LTRRefine;

use 5.014;
use Moose;
use File::Spec;
use File::Find;
use File::Basename;
use File::Copy          qw(move);
use Sort::Naturally     qw(nsort);
use List::UtilsBy       qw(nsort_by);
use List::Util          qw(sum max);
use Cwd                 qw(abs_path);
use Bio::GFF3::LowLevel qw(gff3_parse_feature gff3_format_feature);
use Bio::DB::HTS::Kseq;
use Bio::DB::HTS::Faidx;
use Set::IntervalTree;
use Path::Class::File;
use Carp 'croak';
#use Data::Dump::Color;
use namespace::autoclean;

with 'Tephra::Role::Util', 
     'Tephra::Role::Logger';

=head1 NAME

Tephra::LTR::LTRRefine - Refine LTR predictions based on multiple criteria

=head1 VERSION

Version 0.12.2

=cut

our $VERSION = '0.12.2';
$VERSION = eval $VERSION;

has genome => (
      is       => 'ro',
      isa      => 'Path::Class::File',
      required => 1,
      coerce   => 1,
);

has outfile => (
      is        => 'ro',
      isa       => 'Str',
      predicate => 'has_outfile',
);

has logfile => (
      is        => 'ro',
      isa       => 'Str',
      predicate => 'has_logfile',
);

has n_threshold => (
      is       => 'ro',
      isa      => 'Num',
      required => 0,
      default  => 0.30,
);

has remove_dup_domains => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_remove_dup_domains',
    lazy      => 1,
    default   => 0,
);

has remove_tnp_domains => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_remove_tnp_domains',
    lazy      => 1,
    default   => 0,
);

has domains_required => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_domains_required',
    lazy      => 1,
    default   => 0,
);

has is_trim => (
    is        => 'ro',
    isa       => 'Bool',
    lazy      => 1,
    default   => 0,
);

#
# methods
#
sub collect_features {
    my $self = shift;
    my ($gff_thresh_ref) = @_;
    my ($gff, $pid_thresh) = @{$gff_thresh_ref}{qw(gff pid_threshold)};

    my (%features, %intervals, %coord_map);
    my ($source, $seq_id, $start, $end, $length, $region);

    open my $gffio, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";

    while (my $line = <$gffio>) {
	chomp $line;
	next if $line =~ /^#/;
	my $feature = gff3_parse_feature( $line );
	if ($feature->{type} eq 'repeat_region') {
	    $region = @{$feature->{attributes}{ID}}[0];
	    ($seq_id, $start, $end)  = @{$feature}{qw(seq_id start end)};
	    $length = $end - $start + 1; 
	    my $region_key = join "||", $region."_".$pid_thresh, $start, $end, $length;
	    $features{$seq_id}{$region_key} = [];
	    #$coord_map{$region."_".$pid_thresh} = join "||", $seq_id, $start, $end;
	    $intervals{$region."_".$pid_thresh} = join "||", $start, $end, $length;
	}
	
	if ($feature->{type} ne 'repeat_region') {
	    my $id = @{$feature->{attributes}{ID}}[0];
	    my $parent = @{$feature->{attributes}{Parent}}[0];
	        
	    if ($feature->{start} >= $start && $feature->{end} <= $end) {
		my $region_key = join "||", $region."_".$pid_thresh, $start, $end, $length;
		if ($parent =~ /(repeat_region\d+)/) {
		    my $old_parent = $1;
		    my $new_parent = join "_", $old_parent, $pid_thresh;
		    $parent =~ s/$old_parent/$new_parent/;
		    $feature->{attributes}{Parent}[0] = $parent;
		}
		push @{$features{$seq_id}{$region_key}}, $feature;
	    }
	}
    }

    my ($filtered, $stats) = $self->filter_compound_elements(\%features);
    return ({ collected_features => $filtered, stats => $stats, intervals => \%intervals}); #, region_map => \%coord_map });
}

sub filter_compound_elements {
    my $self = shift;
    my ($features) = @_;
    my (@pdoms, %classII);
{'fraction_non_alphanumeric': 0.1948347383915429, 'fraction_numerical': 0.010169945135822294, 'mean_word_length': 2.770938446014127, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1433688', 'n_tokens_mistral': 2907, 'n_tokens_neox': 2762, 'n_words': 608}