<commit_before>
### Bio::EnsEMBL::Pipeline::RunnableDB::Finished_EST

package Bio::EnsEMBL::Pipeline::RunnableDB::Finished_EST;

use strict;

use Bio::EnsEMBL::Pipeline::RunnableDB;
use Bio::Root::RootI;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Pfetch;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Finished_Pfetch;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Getseqs;
use Bio::EnsEMBL::Pipeline::Runnable::Finished_EST;
use Bio::EnsEMBL::Pipeline::SeqFetcher::OBDAIndexSeqFetcher;
use Bio::EnsEMBL::Pipeline::Config::General;
use Bio::PrimarySeq;

use vars qw(@ISA);
@ISA = qw(Bio::EnsEMBL::Pipeline::RunnableDB);

sub new {
    my ( $new, @args ) = @_;
    my $self = $new->SUPER::new(@args);
    
    # dbobj, input_id, seqfetcher, and analysis objects are all set in
    # in superclass constructor (RunnableDB.pm)
    #print STDERR Dumper($self);
    return $self;
}

sub fetch_input {
    my ($self) = @_;
    
    my @fps;
    
    $self->throw("No input id") unless defined( $self->input_id );
    
    my $contigid = $self->input_id;
    my $rawContigAdaptor = $self->db->get_RawContigAdaptor();
    my $contig   = $rawContigAdaptor->fetch_by_name($contigid);
    
    my $masked   = $contig->get_repeatmasked_seq($PIPELINE_REPEAT_MASKING,$SOFT_MASKING) or $self->throw("Unable to fetch contig");
    # make a Bio::PrimarySeq obj to remove some db intensive queries later on
    my $unmasked = Bio::PrimarySeq->new(-display_id => $contig->display_id(),
					-id         => $contig->id(),
					-seq        => $contig->seq()
					);
    my $seq = $masked->seq;
    if( scalar($seq =~ s/([CATG])/$1/g) > 3 ){
        $self->input_is_void(0);
    }else{
        $self->input_is_void(1);
        $self->warn("Need at least 3 nucleotides");
    }

    my $runnable = Bio::EnsEMBL::Pipeline::Runnable::Finished_EST->new(
        '-query'      => $masked,
        '-unmasked'   => $unmasked,
        '-analysis'   => $self->analysis,
    );
    $self->runnable($runnable);
}

sub runnable {
    my ( $self, $arg ) = @_;
    
    if ( defined($arg) ) {
        $self->throw("[$arg] is not a Bio::EnsEMBL::Pipeline::RunnableI")
        unless $arg->isa("Bio::EnsEMBL::Pipeline::RunnableI");
        
        $self->{_runnable} = $arg;
    }
    
    return $self->{_runnable};
}

sub run {
    my ($self) = @_;
    
    my $runnable = $self->runnable;
    $runnable || $self->throw("Can't run - no runnable object");
    
    $runnable->run;
    
    my $db_version = $runnable->db_version_searched if $runnable->can('db_version_searched');
    $self->db_version_searched($db_version);
    if ( my @output = $runnable->output ) {
	        my $dbobj      = $self->db;
	        my $seqfetcher = Bio::EnsEMBL::Pipeline::SeqFetcher::Finished_Pfetch->new;
	        my %ids        = map { $_->hseqname, 1 } @output;
	        $seqfetcher->write_descriptions( $dbobj, keys(%ids) );        
    }
    <commit_msg>added checking the masked sequence with seg in an attempt to stop 'no valid context' problem<commit_after>
### Bio::EnsEMBL::Pipeline::RunnableDB::Finished_EST

package Bio::EnsEMBL::Pipeline::RunnableDB::Finished_EST;

use strict;

use Bio::EnsEMBL::Pipeline::RunnableDB;
use Bio::Root::RootI;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Pfetch;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Finished_Pfetch;
use Bio::EnsEMBL::Pipeline::SeqFetcher::Getseqs;
use Bio::EnsEMBL::Pipeline::Runnable::Finished_EST;
use Bio::EnsEMBL::Pipeline::SeqFetcher::OBDAIndexSeqFetcher;
use Bio::EnsEMBL::Pipeline::Config::General;
use Bio::PrimarySeq;

use vars qw(@ISA);
@ISA = qw(Bio::EnsEMBL::Pipeline::RunnableDB);

sub new {
    my ( $new, @args ) = @_;
    my $self = $new->SUPER::new(@args);
    
    # dbobj, input_id, seqfetcher, and analysis objects are all set in
    # in superclass constructor (RunnableDB.pm)
    #print STDERR Dumper($self);
    return $self;
}

sub fetch_input {
    my ($self) = @_;
    
    my @fps;
    
    $self->throw("No input id") unless defined( $self->input_id );
    
    my $contigid = $self->input_id;
    my $rawContigAdaptor = $self->db->get_RawContigAdaptor();
    my $contig   = $rawContigAdaptor->fetch_by_name($contigid);
    
    my $masked   = $contig->get_repeatmasked_seq($PIPELINE_REPEAT_MASKING,$SOFT_MASKING) or $self->throw("Unable to fetch contig");
    # make a Bio::PrimarySeq obj to remove some db intensive queries later on
    my $unmasked = Bio::PrimarySeq->new(-display_id => $contig->display_id(),
					-id         => $contig->id(),
					-seq        => $contig->seq()
					);
    my $seq = $masked->seq;
    if( scalar($seq =~ s/([CATG])/$1/g) > 3 ){
        $self->input_is_void(0);
        $self->check_with_seg($masked);
    }else{
        $self->input_is_void(1);
        $self->warn("Need at least 3 nucleotides");
    }

    my $runnable = Bio::EnsEMBL::Pipeline::Runnable::Finished_EST->new(
        '-query'      => $masked,
        '-unmasked'   => $unmasked,
        '-analysis'   => $self->analysis,
    );
    $self->runnable($runnable);
}
sub check_with_seg{
    my ($self, $seqObj_to_test) = @_;

    warn "need a Bio::Seq Obj" unless $seqObj_to_test;

    my ($filename) = $self->_createfiles('/tmp',[qw(seg_checking)]);
    my $file = Bio::SeqIO->new(-file   => ">$filename", 
                               -format => 'Fasta') 
        or $self->throw("Can't create Bio::SeqIO $filename $!");
    $file->write_seq($seqObj_to_test);

    my $seg_cmd = "seg $filename -a";
    my $seg = Bio::SeqIO->new(-file   => "$seg_cmd |",
                              -format => 'Fasta')
        or $self->throw("Can't create Bio::SeqIO $seg_cmd $!");
    my $seq;
    eval{
        $seq = $seg->next_seq->seq;
    };
    unlink($filename);
    if($@){
        $self->throw("There was a problem with SEG masking.\nI tried to '$seg_cmd'");
    }
    if($seq =~ /[CATG]{3}/){
        $self->input_is_void(0);
    }else{
        $self->input_is_void(1);
        $self->warn("Need at least 3 nucleotides after SEG filtering");
    }
    
}
sub _createfiles {
    my ($self, $dirname, $filenames) = @_;
    
    my $unique = {};
    $unique    = { map { $_, $unique->{$_}++ } @$filenames };
    my @files  = ();

    $dirname ||= '/tmp';
    $dirname   =~ s!(\S+)/$!$1!;

    foreach my $file(@$filenames){
        if($unique->{$file}){
            #name not unique add random
            $file .= ".$$.".int(rand(200));
            push(@files, "$dirname/$file");
        }else{
            #name was unique just add it
            push(@files, "$dirname/$file.$$");
        }
    }

    return @files;
}
sub runnable {
    my ( $self, $arg ) = @_;
    
    if ( defined($arg) ) {
        $self->throw("[$arg] is not a Bio::EnsEMBL::Pipeline::RunnableI")
        unless $arg->isa("Bio::EnsEMBL::Pipeline::RunnableI");
        
        $self->{_runnable} = $arg;
    }
    
    return $self->{_runnable};
}

sub run {
    my ($self) = @_;
    
    my $runnable = $self->runnable;
    $runnable || $self->throw("Can't run - no runnable object");
    
    $runnable->run;
    
    my $db_version = $runnable->db_version_searched if $runnable->can('db_version_searched');
    $self->db_version_searched($db_version);
    if ( my @output = $runnable->output ) {
	        my $dbobj      = $self->db;
	        my $seqfetcher = Bio::EnsEMBL::Pipeline::SeqFetcher::Finished_Pfetch->new;
	        my %ids        = map { $_->hseqname, 1 } @output;
	        $seqfetcher->write_descriptions( $dbobj, keys(%ids) );        
    }
    
{'fraction_non_alphanumeric': 0.17703862660944206, 'fraction_numerical': 0.002682403433476395, 'mean_word_length': 2.9350923482849605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1885603', 'n_tokens_mistral': 2811, 'n_tokens_neox': 2673, 'n_words': 505}