<commit_before>package Tephra::Role::GFF;

use 5.014;
use Moose::Role;
use File::Spec;
use File::Basename;
use File::Path          qw(make_path);
use Bio::GFF3::LowLevel qw(gff3_parse_feature);
use Sort::Naturally;
use Path::Class::File;
use Tephra::Alignment::Utils;
#use Data::Dump::Color;
use namespace::autoclean;

=head1 NAME

Tephra::Role::GFF - Utility methods for working with GFF files

=head1 VERSION

Version 0.12.4

=cut

our $VERSION = '0.12.4';
$VERSION = eval $VERSION;

#
# methods
#
sub collect_gff_features {
    my $self = shift;
    my ($gff) = @_;

    my $header;
    open my $in, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";
    while (<$in>) {
	chomp;
	next if /^###$/;
	if (/^##?\w+/) {
	    $header .= $_."\n";
	}
	else {
	    last;
	}
    }
    close $in;
    chomp $header;

    open my $gffio, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";

    my ($seq_id, $start, $end, $region, $key, %features);
    while (my $line = <$gffio>) {
        chomp $line;
        next if $line =~ /^#/;
        my $feature = gff3_parse_feature( $line );
        if ($feature->{type} eq 'repeat_region') {
            $region = @{$feature->{attributes}{ID}}[0];
            ($seq_id, $start, $end) = @{$feature}{qw(seq_id start end)};
	    $key = join "||", $region, $start, $end;

        }
	if ($feature->{type} ne 'repeat_region') {
            if ($feature->{start} >= $start && $feature->{end} <= $end) {
		push @{$features{$seq_id}{$key}}, $feature;
            }
        }
    }
    close $gffio;

    return ($header, \%features);
}

sub get_parent_coords {
    my $self = shift;
    my ($parent, $coord_map) = @_;

    my ($seq_id, $start, $end) = split /\|\|/, $coord_map->{$parent};
    my $pkey = join "||", $parent, $seq_id, $start, $end;

    return ($seq_id, $pkey);
}

sub extract_ltr_sequences {
    my $self = shift;
    my $fasta = $self->genome->absolute->resolve;
    my $gff   = $self->gff->absolute->resolve;
    
    my ($name, $path, $suffix) = fileparse($gff, qr/\.[^.]*/);
    my $dir = File::Spec->catdir($path, $name.'_ltrages');
    unless ( -d $dir ) {
        make_path( $dir, {verbose => 0, mode => 0771,} );
    }

    my ($header, $features) = $self->collect_gff_features($gff);
    my $index = $self->index_ref($fasta);
    
    my ($family, %ltrs, %seen, %coord_map);
    for my $seq_id (keys %$features) {
	for my $rep_region (keys %{$features->{$seq_id}}) {
	    for my $ltr_feature (@{$features->{$seq_id}{$rep_region}}) {
		if ($ltr_feature->{type} =~ /(?:LTR|TRIM|LARD)_retrotransposon/) {
		    my $elem_id = @{$ltr_feature->{attributes}{ID}}[0];
		    $family  = @{$ltr_feature->{attributes}{family}}[0];
		    my ($start, $end) = @{$ltr_feature}{qw(start end)};
		    my $key = join "||", $family, $elem_id, $seq_id, $start, $end;
		    $ltrs{$key}{'full'} = join "||", @{$ltr_feature}{qw(seq_id type start end)};
		    $coord_map{$elem_id} = join "||", @{$ltr_feature}{qw(seq_id start end)};
		}
		elsif ($ltr_feature->{type} eq 'long_terminal_repeat') { # &&
		    #$ltr_feature->{start} == $start || $ltr_feature->{end} == $end) {
		    my $parent = @{$ltr_feature->{attributes}{Parent}}[0];
		    my ($chr_id, $pkey) = $self->get_parent_coords($parent, \%coord_map);
		    my ($ltr_type, $ltr_start, $ltr_end, $ltr_strand) = 
			@{$ltr_feature}{qw(type start end strand)};
		    $ltr_strand //= '?';
		    my $ltrkey = join "||", $chr_id, $ltr_type, $ltr_start, $ltr_end, $ltr_strand;
		    my $parent_key = join "||", $family, $pkey;
		    push @{$ltrs{$parent_key}{'ltrs'}}, $ltrkey; # unless exists $seen{$ltrkey};
		    #$seen{$ltrkey} = 1;
		}
            }
        }
    }
    #dd \%ltrs and exit;

    my @files;
    my $ltrct = 0;
    my $orientation;
    for my $ltr (nsort keys %ltrs) {
        my ($family, $element, $chr, $rstart, $rend) = split /\|\|/, $ltr;
        my ($seq_id, $type, $start, $end) = split /\|\|/, $ltrs{$ltr}{'full'};
        my $ltr_file = join "_", $family, $element, $seq_id, $start, $end, 'ltrs.fasta';
        my $ltrs_out = File::Spec->catfile($dir, $ltr_file);
        die "\n[ERROR]: $ltrs_out exists. This will cause problems downstream. Please remove the previous ".<commit_msg>check if family is defined in LTR GFF3 and define it if not<commit_after>package Tephra::Role::GFF;

use 5.014;
use Moose::Role;
use File::Spec;
use File::Basename;
use File::Path          qw(make_path);
use Bio::GFF3::LowLevel qw(gff3_parse_feature);
use Sort::Naturally;
use Path::Class::File;
use Tephra::Alignment::Utils;
#use Data::Dump::Color;
use namespace::autoclean;

=head1 NAME

Tephra::Role::GFF - Utility methods for working with GFF files

=head1 VERSION

Version 0.12.4

=cut

our $VERSION = '0.12.4';
$VERSION = eval $VERSION;

#
# methods
#
sub collect_gff_features {
    my $self = shift;
    my ($gff) = @_;

    my $header;
    open my $in, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";
    while (<$in>) {
	chomp;
	next if /^###$/;
	if (/^##?\w+/) {
	    $header .= $_."\n";
	}
	else {
	    last;
	}
    }
    close $in;
    chomp $header;

    open my $gffio, '<', $gff or die "\n[ERROR]: Could not open file: $gff\n";

    my ($seq_id, $start, $end, $region, $key, %features);
    while (my $line = <$gffio>) {
        chomp $line;
        next if $line =~ /^#/;
        my $feature = gff3_parse_feature( $line );
        if ($feature->{type} eq 'repeat_region') {
            $region = @{$feature->{attributes}{ID}}[0];
            ($seq_id, $start, $end) = @{$feature}{qw(seq_id start end)};
	    $key = join "||", $region, $start, $end;

        }
	if ($feature->{type} ne 'repeat_region') {
            if ($feature->{start} >= $start && $feature->{end} <= $end) {
		push @{$features{$seq_id}{$key}}, $feature;
            }
        }
    }
    close $gffio;

    return ($header, \%features);
}

sub get_parent_coords {
    my $self = shift;
    my ($parent, $coord_map) = @_;

    my ($seq_id, $start, $end) = split /\|\|/, $coord_map->{$parent};
    my $pkey = join "||", $parent, $seq_id, $start, $end;

    return ($seq_id, $pkey);
}

sub extract_ltr_sequences {
    my $self = shift;
    my $fasta = $self->genome->absolute->resolve;
    my $gff   = $self->gff->absolute->resolve;
    
    my ($name, $path, $suffix) = fileparse($gff, qr/\.[^.]*/);
    my $dir = File::Spec->catdir($path, $name.'_ltrages');
    unless ( -d $dir ) {
        make_path( $dir, {verbose => 0, mode => 0771,} );
    }

    my ($header, $features) = $self->collect_gff_features($gff);
    my $index = $self->index_ref($fasta);
    
    my ($family, %ltrs, %seen, %coord_map);
    for my $seq_id (keys %$features) {
	for my $rep_region (keys %{$features->{$seq_id}}) {
	    for my $ltr_feature (@{$features->{$seq_id}{$rep_region}}) {
		if ($ltr_feature->{type} =~ /(?:LTR|TRIM|LARD)_retrotransposon/) {
		    my $elem_id = @{$ltr_feature->{attributes}{ID}}[0];
		    $family  = @{$ltr_feature->{attributes}{family}}[0];
		    my ($start, $end) = @{$ltr_feature}{qw(start end)};
		    my $key = defined $family ? join "||", $family, $elem_id, $seq_id, $start, $end 
			: join "||", 'RLX', $elem_id, $seq_id, $start, $end;
		    $ltrs{$key}{'full'} = join "||", @{$ltr_feature}{qw(seq_id type start end)};
		    $coord_map{$elem_id} = join "||", @{$ltr_feature}{qw(seq_id start end)};
		}
		elsif ($ltr_feature->{type} eq 'long_terminal_repeat') { # &&
		    #$ltr_feature->{start} == $start || $ltr_feature->{end} == $end) {
		    my $parent = @{$ltr_feature->{attributes}{Parent}}[0];
		    my ($chr_id, $pkey) = $self->get_parent_coords($parent, \%coord_map);
		    my ($ltr_type, $ltr_start, $ltr_end, $ltr_strand) = 
			@{$ltr_feature}{qw(type start end strand)};
		    $ltr_strand //= '?';
		    my $ltrkey = join "||", $chr_id, $ltr_type, $ltr_start, $ltr_end, $ltr_strand;
		    my $parent_key = defined $family ? join "||", $family, $pkey : join "||", 'RLX', $pkey;
		    push @{$ltrs{$parent_key}{'ltrs'}}, $ltrkey; # unless exists $seen{$ltrkey};
		    #$seen{$ltrkey} = 1;
		}
            }
        }
    }
    #dd \%ltrs and exit;

    my @files;
    my $ltrct = 0;
    my $orientation;
    for my $ltr (nsort keys %ltrs) {
        my ($family, $element, $chr, $rstart, $rend) = split /\|\|/, $ltr;
        my ($seq_id, $type, $start, $end) = split /\|\|/, $ltrs{$ltr}{'full'};
        my $ltr_file = join "_", $family, $element, $seq_id, $start, $end, 'ltrs.fasta';
        my $ltrs_out = File::Spec->catfile($dir, $ltr_file);
        die "\n[ERROR]: $ltrs_out exists. This will cause problems downstream. Please remove the previous ".
{'fraction_non_alphanumeric': 0.21310328747959897, 'fraction_numerical': 0.006644905572394498, 'mean_word_length': 3.0697343453510437, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272205', 'n_tokens_mistral': 3529, 'n_tokens_neox': 3350, 'n_words': 759}