<commit_before>#
# You may distribute this module under the same terms as perl itself
#
# POD documentation - main docs before the code

=pod 

=head1 NAME

Bio::EnsEMBL::Pipeline::Runnable::BlastMiniBuilder;

=head1 SYNOPSIS

=head1 DESCRIPTION

=head1 CONTACT

<NAME> <<EMAIL>>

=head1 APPENDIX

The rest of the documentation details each of the object methods. 
Internal methods are usually preceded with a _

=cut

# Let the code begin...

package Bio::EnsEMBL::Pipeline::Runnable::BlastMiniBuilder;

use vars qw(@ISA);
use strict;

use Bio::EnsEMBL::Pipeline::Runnable::MultiMiniGenewise;
use Bio::EnsEMBL::Pipeline::Runnable::Blast;
use Bio::EnsEMBL::Pipeline::Runnable::BlastDB;
use Bio::EnsEMBL::Pipeline::RunnableI;
use Bio::PrimarySeqI;
use Bio::SeqIO;
use Bio::DB::RandomAccessI;
use Bio::EnsEMBL::Pipeline::GeneConf qw (
					 GB_INPUTID_REGEX
					);
use Bio::EnsEMBL::Pipeline::Config::Blast;

@ISA = qw(Bio::EnsEMBL::Pipeline::RunnableI);


=head2 build_runnables

  Args [1]   : @features - list of FeaturePair objects 
               constructed from blast align features.  Within
               each FeaturePair, feature1 and feature2 should
               correspond to the genomic sequence and the
               protein/est/cdna sequence.
  Example    : $self->build_runnables(@features);
  Description: Builds the runnable objects for conducting
               BlastMini(Genewise/Est2Genome) runs.  In 
               the simplest case this
               method creates a single runnable object.  The
               method itself was written to cope with
               situations where the presence of repeated genes
               means that more than one whole gene of high 
               homology is present in a minigenomic sequence.
               This module separates possible repeated genes
               into clusters of exons such that multiple 
               minigenomic sequence fragments can be
               generated for passing to genewise/est2genome.
               When this happens this method returns
               multiple runnables that examine smaller
               portions of genomic DNA.
  Returntype : A list of Bio::EnsEMBL::Pipeline::Runnable
               - the exact identity of the runnable sub-class
               is determined by the calling class (as the 
               actual final runnable construction is done in 
               the inheriting class.)
  Exceptions : None.
  Caller     : $self->run

=cut

sub build_runnables {
  my ($self, @features) = @_;

  my @runnables;

  my %unfiltered_partitioned_features;

  foreach my $raw_feat (@features){
      push (@{$unfiltered_partitioned_features{$raw_feat->hseqname}}, $raw_feat);
  }

  # We partition the features by sequence id such that we can
  # iterate for each id.  Most ids  will fall through
  # to the simplest case where we make a single runnable with one 
  # unique gene per minigenomic sequence fragment.  In cases
  # where multiple similar genes are likely to be present in
  # the minigenomic sequence we switch to the
  # special case algorithm.  When this happens we cluster the 
  # likely exons into groups of similar exons, which we then
  # use make a set of distinct genes.  For each likely gene
  # a minigenomic sequence is made that tightly spans the 
  # likely location of the gene - and most probably excludes
  # any similar genes nearby.  With these smaller minigenomic
  # sequence fragments a runnable is made using ALL of the
  # original blast-derived feature pairs.

  my %partitioned_features;

  foreach my $raw_feat (@features){

    # Imporantly, the features are filtered for low identity matches.
    # This is just for the clustering process.  The full set of features
    # are still passed to the runnable.
    if($raw_feat->percent_id >= 80){
	    push (@{$partitioned_features{$raw_feat->hseqname}}, $raw_feat);
    }
  }

  my %runnable_featids;

  foreach my $seqname (keys %partitioned_features){

    # Preliminary check to see whether our filtered features
    # give enough coverage to make it worth continuing.

    my $coverage = $self->check_coverage($seqname, $partitioned_features{$seqname});

    # If only sparse coverage, go home early.
    unless ($coverage > 0.5) {

      my $runnable = $self->make_object($self->genomic_sequence, $unfiltered_partitioned_features{$seqname});
      push (@runnables, $runnable);

      $runnable_featids{$seqname}++;
      next
    }

    # Now, we attempt to cluster our features into groups of
    # similar exons.

     my $clustered_features = $self->cluster_features(\@{$partitioned_features{$seqname}});

    # We have to do a little test to see if the clusters
    # represent the bulk of the features present in the 
    # genomic region.  This test helps avoid the situation
    # where a non-repeated gene has exons that are very 
    # similar to one another.  If the flag $clusters_seem_real
    # is not set to 1, the the analysis will default to the  
    # non-repeated gene way of doing things.

    my $clusters_seem_real = 0;

    if (@$clustered_features) {
      my $number_of_clustered_features = 0;
      foreach my $cluster (@$clustered_features) {
	foreach my $clust_feat (@$cluster){
	  $number_of_clustered_features++;
	}
      }

      if ($number_of_clustered_features  >= (0.7 * (scalar @{$partitioned_features{$seqname}}))) {
	$clusters_seem_real = 1;
      }
    }

    # If we have clusters in such numbers that it looks like
    # there might be repeated genes, we break the mini genomic
    # sequence into fragments and construct multiple runnables.
    # Otherwise, we default to the normal way of building our
    # runnable.
    
    if ((@$clustered_features)&&($clusters_seem_real > 0)) {

      print STDERR "Minigenomic sequence could contain a number "
	. "of highly similar genes.  Fragmenting the minigenomic "
	  . "sequence to try to resolve these genes individually.\n";

      my $gene_clusters = $self->form_gene_clusters($clustered_features);

      # Determine the extreme start and ends of each gene
      # cluster.  Decide on positions to split the genomic
      # sequence for genomewise - mid-way between 
      # ends and starts of neighboring clusters.  The
      # fragments from clusters that don't have the maximum
      # number of members are a problem - these presumably 
      # will fall at the ends of the genomic fragment and 
      # should just be discarded.

    GENE:      
      foreach my $gene_cluster (@$gene_clusters){

	my @sorted_gene_cluster = sort {$a->{_gsf_start} <=> $b->{_gsf_start};} @$gene_cluster;

	my $rough_gene_length = $_sorted_gene_cluster[-1]->{_gsf_end} - $sorted_gene_cluster[0]->{_gsf_start};

	my $padding_length;
	if ($rough_gene_length > 10000) {
	  $padding_length = 10000;
	} else {
	  $padding_length = 1000;
	}

       	my $cluster_start = $sorted_gene_cluster[0]->{_gsf_start} - $padding_length;
	my $cluster_end = $sorted_gene_cluster[-1]->{_gsf_end} + $padding_length;

	#print "Cluster start: $cluster_start   Cluster end: $cluster_end\n";

        if ($cluster_start < 1) {
	  if ($sorted_gene_cluster[0]->{_gsf_end} > 0) {
	    $cluster_start = 1;
	  }
	}
        if ($cluster_end > $self->genomic_sequence->length) {
	  if ($sorted_gene_cluster[0]->{_gsf_start} <= $self->genomic_sequence->length) {
	    $cluster_end = $self->genomic_sequence->length;
	  } 
	}        

	# Here we create our genomic fragment for passing to 
	# our runnable.  This is the fragment of genomic sequence
	# that we have calculated should only contain the 
	# exons of one single gene.  It is padded with Ns to
        # maintain the coordinate system of the features that
	# may be returned.  Another way of looking at this is
	# that we make a genomic sequence where the regions <commit_msg>Fixed typo<commit_after>#
# You may distribute this module under the same terms as perl itself
#
# POD documentation - main docs before the code

=pod 

=head1 NAME

Bio::EnsEMBL::Pipeline::Runnable::BlastMiniBuilder;

=head1 SYNOPSIS

=head1 DESCRIPTION

=head1 CONTACT

<NAME> <<EMAIL>>

=head1 APPENDIX

The rest of the documentation details each of the object methods. 
Internal methods are usually preceded with a _

=cut

# Let the code begin...

package Bio::EnsEMBL::Pipeline::Runnable::BlastMiniBuilder;

use vars qw(@ISA);
use strict;

use Bio::EnsEMBL::Pipeline::Runnable::MultiMiniGenewise;
use Bio::EnsEMBL::Pipeline::Runnable::Blast;
use Bio::EnsEMBL::Pipeline::Runnable::BlastDB;
use Bio::EnsEMBL::Pipeline::RunnableI;
use Bio::PrimarySeqI;
use Bio::SeqIO;
use Bio::DB::RandomAccessI;
use Bio::EnsEMBL::Pipeline::GeneConf qw (
					 GB_INPUTID_REGEX
					);
use Bio::EnsEMBL::Pipeline::Config::Blast;

@ISA = qw(Bio::EnsEMBL::Pipeline::RunnableI);


=head2 build_runnables

  Args [1]   : @features - list of FeaturePair objects 
               constructed from blast align features.  Within
               each FeaturePair, feature1 and feature2 should
               correspond to the genomic sequence and the
               protein/est/cdna sequence.
  Example    : $self->build_runnables(@features);
  Description: Builds the runnable objects for conducting
               BlastMini(Genewise/Est2Genome) runs.  In 
               the simplest case this
               method creates a single runnable object.  The
               method itself was written to cope with
               situations where the presence of repeated genes
               means that more than one whole gene of high 
               homology is present in a minigenomic sequence.
               This module separates possible repeated genes
               into clusters of exons such that multiple 
               minigenomic sequence fragments can be
               generated for passing to genewise/est2genome.
               When this happens this method returns
               multiple runnables that examine smaller
               portions of genomic DNA.
  Returntype : A list of Bio::EnsEMBL::Pipeline::Runnable
               - the exact identity of the runnable sub-class
               is determined by the calling class (as the 
               actual final runnable construction is done in 
               the inheriting class.)
  Exceptions : None.
  Caller     : $self->run

=cut

sub build_runnables {
  my ($self, @features) = @_;

  my @runnables;

  my %unfiltered_partitioned_features;

  foreach my $raw_feat (@features){
      push (@{$unfiltered_partitioned_features{$raw_feat->hseqname}}, $raw_feat);
  }

  # We partition the features by sequence id such that we can
  # iterate for each id.  Most ids  will fall through
  # to the simplest case where we make a single runnable with one 
  # unique gene per minigenomic sequence fragment.  In cases
  # where multiple similar genes are likely to be present in
  # the minigenomic sequence we switch to the
  # special case algorithm.  When this happens we cluster the 
  # likely exons into groups of similar exons, which we then
  # use make a set of distinct genes.  For each likely gene
  # a minigenomic sequence is made that tightly spans the 
  # likely location of the gene - and most probably excludes
  # any similar genes nearby.  With these smaller minigenomic
  # sequence fragments a runnable is made using ALL of the
  # original blast-derived feature pairs.

  my %partitioned_features;

  foreach my $raw_feat (@features){

    # Imporantly, the features are filtered for low identity matches.
    # This is just for the clustering process.  The full set of features
    # are still passed to the runnable.
    if($raw_feat->percent_id >= 80){
	    push (@{$partitioned_features{$raw_feat->hseqname}}, $raw_feat);
    }
  }

  my %runnable_featids;

  foreach my $seqname (keys %partitioned_features){

    # Preliminary check to see whether our filtered features
    # give enough coverage to make it worth continuing.

    my $coverage = $self->check_coverage($seqname, $partitioned_features{$seqname});

    # If only sparse coverage, go home early.
    unless ($coverage > 0.5) {

      my $runnable = $self->make_object($self->genomic_sequence, $unfiltered_partitioned_features{$seqname});
      push (@runnables, $runnable);

      $runnable_featids{$seqname}++;
      next
    }

    # Now, we attempt to cluster our features into groups of
    # similar exons.

     my $clustered_features = $self->cluster_features(\@{$partitioned_features{$seqname}});

    # We have to do a little test to see if the clusters
    # represent the bulk of the features present in the 
    # genomic region.  This test helps avoid the situation
    # where a non-repeated gene has exons that are very 
    # similar to one another.  If the flag $clusters_seem_real
    # is not set to 1, the the analysis will default to the  
    # non-repeated gene way of doing things.

    my $clusters_seem_real = 0;

    if (@$clustered_features) {
      my $number_of_clustered_features = 0;
      foreach my $cluster (@$clustered_features) {
	foreach my $clust_feat (@$cluster){
	  $number_of_clustered_features++;
	}
      }

      if ($number_of_clustered_features  >= (0.7 * (scalar @{$partitioned_features{$seqname}}))) {
	$clusters_seem_real = 1;
      }
    }

    # If we have clusters in such numbers that it looks like
    # there might be repeated genes, we break the mini genomic
    # sequence into fragments and construct multiple runnables.
    # Otherwise, we default to the normal way of building our
    # runnable.
    
    if ((@$clustered_features)&&($clusters_seem_real > 0)) {

      print STDERR "Minigenomic sequence could contain a number "
	. "of highly similar genes.  Fragmenting the minigenomic "
	  . "sequence to try to resolve these genes individually.\n";

      my $gene_clusters = $self->form_gene_clusters($clustered_features);

      # Determine the extreme start and ends of each gene
      # cluster.  Decide on positions to split the genomic
      # sequence for genomewise - mid-way between 
      # ends and starts of neighboring clusters.  The
      # fragments from clusters that don't have the maximum
      # number of members are a problem - these presumably 
      # will fall at the ends of the genomic fragment and 
      # should just be discarded.

    GENE:      
      foreach my $gene_cluster (@$gene_clusters){

	my @sorted_gene_cluster = sort {$a->{_gsf_start} <=> $b->{_gsf_start};} @$gene_cluster;

	my $rough_gene_length = $sorted_gene_cluster[-1]->{_gsf_end} - $sorted_gene_cluster[0]->{_gsf_start};

	my $padding_length;
	if ($rough_gene_length > 10000) {
	  $padding_length = 10000;
	} else {
	  $padding_length = 1000;
	}

       	my $cluster_start = $sorted_gene_cluster[0]->{_gsf_start} - $padding_length;
	my $cluster_end = $sorted_gene_cluster[-1]->{_gsf_end} + $padding_length;

	#print "Cluster start: $cluster_start   Cluster end: $cluster_end\n";

        if ($cluster_start < 1) {
	  if ($sorted_gene_cluster[0]->{_gsf_end} > 0) {
	    $cluster_start = 1;
	  }
	}
        if ($cluster_end > $self->genomic_sequence->length) {
	  if ($sorted_gene_cluster[0]->{_gsf_start} <= $self->genomic_sequence->length) {
	    $cluster_end = $self->genomic_sequence->length;
	  } 
	}        

	# Here we create our genomic fragment for passing to 
	# our runnable.  This is the fragment of genomic sequence
	# that we have calculated should only contain the 
	# exons of one single gene.  It is padded with Ns to
        # maintain the coordinate system of the features that
	# may be returned.  Another way of looking at this is
	# that we make a genomic sequence where the regions 
{'fraction_non_alphanumeric': 0.08171106557377049, 'fraction_numerical': 0.005763319672131148, 'mean_word_length': 3.105415352260778, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5933371', 'n_tokens_mistral': 4820, 'n_tokens_neox': 4459, 'n_words': 1639}