<commit_before>=head1 LICENSE

Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut


=head1 CONTACT

Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.

=head1 NAME

Bio::EnsEMBL::Compara::SeqMember

=head1 DESCRIPTION

Class to represent a member that has a sequence attached.
It is currently used for proteins and RNAs.

=head1 INHERITANCE TREE

  Bio::EnsEMBL::Compara::SeqMember
  `- Bio::EnsEMBL::Compara::Member

=head1 SYNOPSIS

Member properties:
 - seq_member_id() is an alias for dbID()

Accessors to the sequence (s):
 - sequence() and sequence_id()
 - seq_length()
 - other_sequence()
 - bioseq()

Links with the Ensembl Core objects:
 - get_Transcript()
 - get_Translation()

Links with other Ensembl Compara Member objects:
 - gene_member() and gene_member_id()

=head1 AUTHORSHIP

Ensembl Team. Individual contributions can be found in the GIT log.

=head1 APPENDIX

The rest of the documentation details each of the object methods.
Internal methods are usually preceded with an underscore (_)

=head1 METHODS

=cut


package Bio::EnsEMBL::Compara::SeqMember;

use strict;
#use warnings;

#use feature qw(switch);

use Bio::Seq;

use Bio::EnsEMBL::Utils::Argument;
use Bio::EnsEMBL::Utils::Exception;
use Bio::EnsEMBL::Utils::Scalar qw(:all);


use base ('Bio::EnsEMBL::Compara::Member');


=head2 new (CONSTRUCTOR)

    Arg [-SEQUENCE_ID] (opt) : int
        the $sequence_id for the sequence table in the database
    Example :
	my $peptide = new Bio::EnsEMBL::Compara::SeqMember(-sequence_id => $seq_id);
    Description: Creates a new SeqMember object
    Returntype : Bio::EnsEMBL::Compara::SeqMember
    Exceptions : none

=cut

sub new {
  my ($class, @args) = @_;

  my $self = $class->SUPER::new(@args);
  
  if (scalar @args) {
    my ($sequence_id, $sequence) = rearrange([qw(SEQUENCE_ID SEQUENCE)], @args);

    $sequence_id && $self->sequence_id($sequence_id);
    $sequence && $self->sequence($sequence);
  }

  return $self;
}


=head2 copy

  Arg [1]    : object $source_object (optional)
  Example    : my $member_copy = $member->copy();
  Description: copies the object, optionally by topping up a given structure (to support multiple inheritance)
  Returntype : Bio::EnsEMBL::Compara::SeqMember
  Exceptions : none

=cut

sub copy {
  my $self = shift;
  
  my $mycopy = $self->SUPER::copy(@_);
  
  $mycopy->sequence($self->sequence);
  $mycopy->sequence_id($self->sequence_id);
  $mycopy->gene_member_id($self->gene_member_id);
  
  return $mycopy;
}



=head2 new_from_Transcript

  Arg [-TRANSCRIPT] : Bio::EnsEMBL::Transcript
  Arg [-GENOME_DB] : Bio::EnsEMBL::Compara::GenomeDB
  Arg [-TRANSLATE] : boolean: whether the transcript should be translated
  Example    : $member = Bio::EnsEMBL::Compara::SeqMember->new_from_Transcript(
                  $transcript, $genome_db, 'translate');
  Description: contructor method which takes an Ensembl::Gene object
               and Compara:GenomeDB object and creates a new SeqMember object
               translating from the Gene object
  Returntype : Bio::Ensembl::Compara::SeqMember
  Exceptions :

=cut

sub new_from_Transcript {
    my ($class, @args) = @_;

    my ($transcript, $genome_db, $translate) = rearrange([qw(TRANSCRIPT GENOME_DB TRANSLATE)], @args);

    assert_ref($transcript, 'Bio::EnsEMBL::Transcript');

    my $seq_string;
    my ($start, $end) = ($transcript->seq_region_start, $transcript->seq_region_end);
    my $stable_id = $transcript->stable_id ||
      throw("COREDB error: does not contain transcript stable id for transcript_id ".$transcript->dbID."\n");

    if ($translate) {
        my ($start, $end) = ($transcript->coding_region_start, $transcript->coding_region_end);

        if(not defined($transcript->translation)) {
            throw("request to translate a transcript without a defined translation", $transcript->stable_id);
        }

        $stable_id = $transcript->translation->stable_id ||
            throw("COREDB error: does not contain translation stable id for translation_id ".$transcript->translation->dbID."\n");

        $seq_string = $transcript->translation->seq;

        if ($seq_string =~ /^X+$/) {
            warn("X+ in sequence from translation " . $transcript->translation->stable_id."\n");
        } elsif (length($seq_string) == 0) {
            warn("zero length sequence from translation " . $transcript->translation->stable_id."\n");
        }

    } else {
        unless ($seq_string = $transcript->spliced_seq) {
            throw("COREDB error: unable to get a BioSeq spliced_seq from ". $transcript->stable_id);
        }
        if (length($seq_string) == 0) {
            warn("zero length sequence from transcript " . $transcript->stable_id."\n");
        }
    }

    my $seq_member = Bio::EnsEMBL::Compara::SeqMember->new(
        -STABLE_ID => $stable_id,
        -DISPLAY_LABEL => ($transcript->display_xref ? $transcript->display_xref->display_id : undef),
        -DNAFRAG_START => $start,
        -DNAFRAG_END => $end,
        -DNAFRAG_STRAND => $transcript->seq_region_strand,

        -DNAFRAG => $genome_db->adaptor->db->get_DnaFragAdaptor->fetch_by_GenomeDB_and_name($genome_db, $transcript->seq_region_name),
        -GENOME_DB_ID => $genome_db->dbID,
        -TAXON_ID => $genome_db->taxon_id,

        -SOURCE_NAME => ($translate ? 'ENSEMBLPEP' : 'ENSEMBLTRANS'),
        -SEQUENCE => $seq_string,
        -DESCRIPTION => $transcript->description,
    );
    $seq_member->{core_transcript} = $transcript;
    return $seq_member;
}


sub member_id { ## DEPRECATED
  my $self = shift;
  deprecate('SeqMember::member_id() is deprecated and will be removed in e79. Please use seq_member_id() instead');
  return $self->dbID(@_);
}


=head2 seq_member_id

  Arg [1]    : (opt) integer
  Description: alias for dbID()

=cut

sub seq_member_id {
  my $self = shift;
  return $self->dbID(@_);
}



#
# Sequence methods
#####################



=head2 sequence

  Arg [1]    : (opt) string $sequence
  Example    : my $seq = $member->sequence;
  Description: Get/set the sequence string of this member
               Will lazy load by sequence_id if needed and able
  Returntype : string
  Exceptions : none
  Caller     : general

=cut

sub sequence {
  my $self = shift;
  my $sequence = shift;
  if(defined $sequence) {
    $self->{'_seq_length'} = undef;
    $self->{'_sequence'} = $sequence;
    $self->{'_seq_length'} = length($self->{'_sequence'}) if(defined($self->{'_sequence'}));
    return $self->{'_sequence'};
  }
  
  if(!defined($self->{'_sequence'}) and
     defined($self->sequence_id()) and     
     defined($self->adaptor))
  {
    $self->{'_sequence'} = $self->adaptor->db->get_SequenceAdaptor->fetch_by_dbID($self->sequence_id);
    $self->{'_seq_length'} = length($self->{'_sequence'}) if(defined($self->{'_sequence'}));
  }

  return $self->{'_sequence'};
}


=head2 other_sequence

  Arg [1]    : string $seq_type
  Arg [2]    : (opt) string $sequence
  Example    : my $filtered_seq = $member->other_sequence('filtered');
  Description: Get/Set the alternative sequence of type $seq_type for this member.
                Currently, proteins have 'cds', 'exon_bounded' and 'exon_cased'
                sequences, while RNAs have 'seq_with_flanking' sequences.
                The undef $seq_type maps to the default sequence ($member->sequence())
                If $sequence is set: store it in the database.
                'exon_cased' maps to the sequence string of this member with alternating upper
                and lower case corresponding to the translateable exons.
                'exon_bounded' maps to the sequence string of this member with exon boundaries
                denoted as O, B, or J depending on the phase (O=0, B=1, J=2)
  Returntype : string
  Exceptions : none
  Caller     : general

=cut

sub other_sequence {
    my $self = shift;
    my $seq_type = shift;
    my $sequence = shift;

    # Defaults to "->sequence()"
    return $self->sequence($sequence) unless $seq_type;

    my $key = "_sequence_$seq_type";

    # Called as a setter
    if (defined $sequence) {
        $self->adaptor->db->get_SequenceAdaptor->store_other_sequence($self, $sequence, $seq_type);
        $self->{$key} = $sequence;
    }

    # First option, we look in the compara db
    if (not defined $self->{$key}) {
        $self->{$key} = $self->adaptor->db->get_SequenceAdaptor->fetch_other_sequence_by_member_id_type($self->seq_member_id, $seq_type);
    }

    # Second option, we build the sequence from the core db
    if (not defined $self->{$key}) {
        if ($seq_type eq 'cds') {
            $self->_prepare_cds_sequence;
        } elsif ($seq_type =~ /^exon_/) {
            $self->_prepare_exon_sequences;
        }
    }

    return $self->{$key};
}

# This method gets the CDS of a peptide
sub _prepare_cds_sequence {
    my $self = shift;

    die "ncRNA transcripts don't have CDS sequences. Their nucleotide sequence is directly accessible with SeqMember::sequence().\n" if $self->source_name =~ /TRANS$/;
    die "Uniprot entries don't have CDS sequences. They are only defined at the protein level.\n" if $self->source_name =~ m/^Uniprot/;

    if ($self->source_name eq 'ENSEMBLPEP') {
        $self->{_sequence_cds} = $self->get_Transcript->translateable_seq;
    } else {
        warn "SeqMember doesn't know how to get a CDS sequence for ", $self->source_name;
        $self->{_sequence_cds} = '';
    }
}

# This method gets the exons of the peptide, and builds the exon_cased and exon_bounded sequences
# Given that it is quite slow, all the exon-based sequences should be computed here
sub _prepare_exon_sequences {
    my $self = shift;

    # If there is the exon_bounded sequence, it is only a matter of splitting it and alternating the case
    my $exon_bounded_seq = $self->{_sequence_exon_bounded};
    $exon_bounded_seq = $self->adaptor->db->get_SequenceAdaptor->fetch_other_sequence_by_member_id_type($self->seq_member_id, 'exon_bounded') unless $exon_bounded_seq;

    if ($exon_bounded_seq) {
        $self->{_sequence_exon_bounded} = $exon_bounded_seq;
        my $i = 0;
        $self->{_sequence_exon_cased} = join('', map {$i++%2 ? lc($_) : $_} split( /[boj]/, $exon_bounded_seq));

    } else {

        my $sequence = $self->sequence;
        my $transcript = $self->get_Transcript;
        my @exons = @{$transcript->get_all_translateable_Exons};
        # @exons probably doesn't match the protein if there are such edits
        my @seq_edits = @{$transcript->translation->get_all_SeqEdits('amino_acid_sub')};
        push @seq_edits, @{$transcript->get_all_SeqEdits('_rna_edit')};

        if (((scalar @exons) == 1) or (scalar(@seq_edits) > 0)) {
            $self->{_sequence_exon_cased} = $sequence;
            $self->{_sequence_exon_bounded} = $sequence;
            return;
        }

        # Otherwise, we have to parse the exons
        my %boundary_chars = (0 => 'o', 1 => 'b', 2 => 'j');
        my $left_over = $exons[0]->phase > 0 ? -$exons[0]->phase : 0;
        my @this_seq = ();
        my @exon_sequences = ();
        foreach my $exon (@exons) {
            my $exon_pep_len = POSIX::ceil(($exon->length - $left_over) / 3);
            my $exon_seq = substr($sequence, 0, $exon_pep_len, '');
            $left_over += 3*$exon_pep_len - $exon->length;
            #printf("%s: exon of len %d -> phase %d: %s\n", $transcript->stable_id, $exon_pep_len, $left_over, $exon_seq);
            push @this_seq, $exon_seq;<commit_msg>For some species, like barrel medic, there can be 0 translateable exons<commit_after>=head1 LICENSE

Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut


=head1 CONTACT

Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.

=head1 NAME

Bio::EnsEMBL::Compara::SeqMember

=head1 DESCRIPTION

Class to represent a member that has a sequence attached.
It is currently used for proteins and RNAs.

=head1 INHERITANCE TREE

  Bio::EnsEMBL::Compara::SeqMember
  `- Bio::EnsEMBL::Compara::Member

=head1 SYNOPSIS

Member properties:
 - seq_member_id() is an alias for dbID()

Accessors to the sequence (s):
 - sequence() and sequence_id()
 - seq_length()
 - other_sequence()
 - bioseq()

Links with the Ensembl Core objects:
 - get_Transcript()
 - get_Translation()

Links with other Ensembl Compara Member objects:
 - gene_member() and gene_member_id()

=head1 AUTHORSHIP

Ensembl Team. Individual contributions can be found in the GIT log.

=head1 APPENDIX

The rest of the documentation details each of the object methods.
Internal methods are usually preceded with an underscore (_)

=head1 METHODS

=cut


package Bio::EnsEMBL::Compara::SeqMember;

use strict;
#use warnings;

#use feature qw(switch);

use Bio::Seq;

use Bio::EnsEMBL::Utils::Argument;
use Bio::EnsEMBL::Utils::Exception;
use Bio::EnsEMBL::Utils::Scalar qw(:all);


use base ('Bio::EnsEMBL::Compara::Member');


=head2 new (CONSTRUCTOR)

    Arg [-SEQUENCE_ID] (opt) : int
        the $sequence_id for the sequence table in the database
    Example :
	my $peptide = new Bio::EnsEMBL::Compara::SeqMember(-sequence_id => $seq_id);
    Description: Creates a new SeqMember object
    Returntype : Bio::EnsEMBL::Compara::SeqMember
    Exceptions : none

=cut

sub new {
  my ($class, @args) = @_;

  my $self = $class->SUPER::new(@args);
  
  if (scalar @args) {
    my ($sequence_id, $sequence) = rearrange([qw(SEQUENCE_ID SEQUENCE)], @args);

    $sequence_id && $self->sequence_id($sequence_id);
    $sequence && $self->sequence($sequence);
  }

  return $self;
}


=head2 copy

  Arg [1]    : object $source_object (optional)
  Example    : my $member_copy = $member->copy();
  Description: copies the object, optionally by topping up a given structure (to support multiple inheritance)
  Returntype : Bio::EnsEMBL::Compara::SeqMember
  Exceptions : none

=cut

sub copy {
  my $self = shift;
  
  my $mycopy = $self->SUPER::copy(@_);
  
  $mycopy->sequence($self->sequence);
  $mycopy->sequence_id($self->sequence_id);
  $mycopy->gene_member_id($self->gene_member_id);
  
  return $mycopy;
}



=head2 new_from_Transcript

  Arg [-TRANSCRIPT] : Bio::EnsEMBL::Transcript
  Arg [-GENOME_DB] : Bio::EnsEMBL::Compara::GenomeDB
  Arg [-TRANSLATE] : boolean: whether the transcript should be translated
  Example    : $member = Bio::EnsEMBL::Compara::SeqMember->new_from_Transcript(
                  $transcript, $genome_db, 'translate');
  Description: contructor method which takes an Ensembl::Gene object
               and Compara:GenomeDB object and creates a new SeqMember object
               translating from the Gene object
  Returntype : Bio::Ensembl::Compara::SeqMember
  Exceptions :

=cut

sub new_from_Transcript {
    my ($class, @args) = @_;

    my ($transcript, $genome_db, $translate) = rearrange([qw(TRANSCRIPT GENOME_DB TRANSLATE)], @args);

    assert_ref($transcript, 'Bio::EnsEMBL::Transcript');

    my $seq_string;
    my ($start, $end) = ($transcript->seq_region_start, $transcript->seq_region_end);
    my $stable_id = $transcript->stable_id ||
      throw("COREDB error: does not contain transcript stable id for transcript_id ".$transcript->dbID."\n");

    if ($translate) {
        my ($start, $end) = ($transcript->coding_region_start, $transcript->coding_region_end);

        if(not defined($transcript->translation)) {
            throw("request to translate a transcript without a defined translation", $transcript->stable_id);
        }

        $stable_id = $transcript->translation->stable_id ||
            throw("COREDB error: does not contain translation stable id for translation_id ".$transcript->translation->dbID."\n");

        $seq_string = $transcript->translation->seq;

        if ($seq_string =~ /^X+$/) {
            warn("X+ in sequence from translation " . $transcript->translation->stable_id."\n");
        } elsif (length($seq_string) == 0) {
            warn("zero length sequence from translation " . $transcript->translation->stable_id."\n");
        }

    } else {
        unless ($seq_string = $transcript->spliced_seq) {
            throw("COREDB error: unable to get a BioSeq spliced_seq from ". $transcript->stable_id);
        }
        if (length($seq_string) == 0) {
            warn("zero length sequence from transcript " . $transcript->stable_id."\n");
        }
    }

    my $seq_member = Bio::EnsEMBL::Compara::SeqMember->new(
        -STABLE_ID => $stable_id,
        -DISPLAY_LABEL => ($transcript->display_xref ? $transcript->display_xref->display_id : undef),
        -DNAFRAG_START => $start,
        -DNAFRAG_END => $end,
        -DNAFRAG_STRAND => $transcript->seq_region_strand,

        -DNAFRAG => $genome_db->adaptor->db->get_DnaFragAdaptor->fetch_by_GenomeDB_and_name($genome_db, $transcript->seq_region_name),
        -GENOME_DB_ID => $genome_db->dbID,
        -TAXON_ID => $genome_db->taxon_id,

        -SOURCE_NAME => ($translate ? 'ENSEMBLPEP' : 'ENSEMBLTRANS'),
        -SEQUENCE => $seq_string,
        -DESCRIPTION => $transcript->description,
    );
    $seq_member->{core_transcript} = $transcript;
    return $seq_member;
}


sub member_id { ## DEPRECATED
  my $self = shift;
  deprecate('SeqMember::member_id() is deprecated and will be removed in e79. Please use seq_member_id() instead');
  return $self->dbID(@_);
}


=head2 seq_member_id

  Arg [1]    : (opt) integer
  Description: alias for dbID()

=cut

sub seq_member_id {
  my $self = shift;
  return $self->dbID(@_);
}



#
# Sequence methods
#####################



=head2 sequence

  Arg [1]    : (opt) string $sequence
  Example    : my $seq = $member->sequence;
  Description: Get/set the sequence string of this member
               Will lazy load by sequence_id if needed and able
  Returntype : string
  Exceptions : none
  Caller     : general

=cut

sub sequence {
  my $self = shift;
  my $sequence = shift;
  if(defined $sequence) {
    $self->{'_seq_length'} = undef;
    $self->{'_sequence'} = $sequence;
    $self->{'_seq_length'} = length($self->{'_sequence'}) if(defined($self->{'_sequence'}));
    return $self->{'_sequence'};
  }
  
  if(!defined($self->{'_sequence'}) and
     defined($self->sequence_id()) and     
     defined($self->adaptor))
  {
    $self->{'_sequence'} = $self->adaptor->db->get_SequenceAdaptor->fetch_by_dbID($self->sequence_id);
    $self->{'_seq_length'} = length($self->{'_sequence'}) if(defined($self->{'_sequence'}));
  }

  return $self->{'_sequence'};
}


=head2 other_sequence

  Arg [1]    : string $seq_type
  Arg [2]    : (opt) string $sequence
  Example    : my $filtered_seq = $member->other_sequence('filtered');
  Description: Get/Set the alternative sequence of type $seq_type for this member.
                Currently, proteins have 'cds', 'exon_bounded' and 'exon_cased'
                sequences, while RNAs have 'seq_with_flanking' sequences.
                The undef $seq_type maps to the default sequence ($member->sequence())
                If $sequence is set: store it in the database.
                'exon_cased' maps to the sequence string of this member with alternating upper
                and lower case corresponding to the translateable exons.
                'exon_bounded' maps to the sequence string of this member with exon boundaries
                denoted as O, B, or J depending on the phase (O=0, B=1, J=2)
  Returntype : string
  Exceptions : none
  Caller     : general

=cut

sub other_sequence {
    my $self = shift;
    my $seq_type = shift;
    my $sequence = shift;

    # Defaults to "->sequence()"
    return $self->sequence($sequence) unless $seq_type;

    my $key = "_sequence_$seq_type";

    # Called as a setter
    if (defined $sequence) {
        $self->adaptor->db->get_SequenceAdaptor->store_other_sequence($self, $sequence, $seq_type);
        $self->{$key} = $sequence;
    }

    # First option, we look in the compara db
    if (not defined $self->{$key}) {
        $self->{$key} = $self->adaptor->db->get_SequenceAdaptor->fetch_other_sequence_by_member_id_type($self->seq_member_id, $seq_type);
    }

    # Second option, we build the sequence from the core db
    if (not defined $self->{$key}) {
        if ($seq_type eq 'cds') {
            $self->_prepare_cds_sequence;
        } elsif ($seq_type =~ /^exon_/) {
            $self->_prepare_exon_sequences;
        }
    }

    return $self->{$key};
}

# This method gets the CDS of a peptide
sub _prepare_cds_sequence {
    my $self = shift;

    die "ncRNA transcripts don't have CDS sequences. Their nucleotide sequence is directly accessible with SeqMember::sequence().\n" if $self->source_name =~ /TRANS$/;
    die "Uniprot entries don't have CDS sequences. They are only defined at the protein level.\n" if $self->source_name =~ m/^Uniprot/;

    if ($self->source_name eq 'ENSEMBLPEP') {
        $self->{_sequence_cds} = $self->get_Transcript->translateable_seq;
    } else {
        warn "SeqMember doesn't know how to get a CDS sequence for ", $self->source_name;
        $self->{_sequence_cds} = '';
    }
}

# This method gets the exons of the peptide, and builds the exon_cased and exon_bounded sequences
# Given that it is quite slow, all the exon-based sequences should be computed here
sub _prepare_exon_sequences {
    my $self = shift;

    # If there is the exon_bounded sequence, it is only a matter of splitting it and alternating the case
    my $exon_bounded_seq = $self->{_sequence_exon_bounded};
    $exon_bounded_seq = $self->adaptor->db->get_SequenceAdaptor->fetch_other_sequence_by_member_id_type($self->seq_member_id, 'exon_bounded') unless $exon_bounded_seq;

    if ($exon_bounded_seq) {
        $self->{_sequence_exon_bounded} = $exon_bounded_seq;
        my $i = 0;
        $self->{_sequence_exon_cased} = join('', map {$i++%2 ? lc($_) : $_} split( /[boj]/, $exon_bounded_seq));

    } else {

        my $sequence = $self->sequence;
        my $transcript = $self->get_Transcript;
        my @exons = @{$transcript->get_all_translateable_Exons};
        # @exons probably doesn't match the protein if there are such edits
        my @seq_edits = @{$transcript->translation->get_all_SeqEdits('amino_acid_sub')};
        push @seq_edits, @{$transcript->get_all_SeqEdits('_rna_edit')};

        if (((scalar @exons) <= 1) or (scalar(@seq_edits) > 0)) {
            $self->{_sequence_exon_cased} = $sequence;
            $self->{_sequence_exon_bounded} = $sequence;
            return;
        }

        # Otherwise, we have to parse the exons
        my %boundary_chars = (0 => 'o', 1 => 'b', 2 => 'j');
        my $left_over = $exons[0]->phase > 0 ? -$exons[0]->phase : 0;
        my @this_seq = ();
        my @exon_sequences = ();
        foreach my $exon (@exons) {
            my $exon_pep_len = POSIX::ceil(($exon->length - $left_over) / 3);
            my $exon_seq = substr($sequence, 0, $exon_pep_len, '');
            $left_over += 3*$exon_pep_len - $exon->length;
            #printf("%s: exon of len %d -> phase %d: %s\n", $transcript->stable_id, $exon_pep_len, $left_over, $exon_seq);
            push @this_seq, $exon_seq;
{'fraction_non_alphanumeric': 0.13062953009202705, 'fraction_numerical': 0.0043570323316230964, 'mean_word_length': 3.3668207681365576, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 279, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2126093', 'n_tokens_mistral': 8243, 'n_tokens_neox': 7729, 'n_words': 2144}