<commit_before>package CoGe::Accessory::GenBank;

use strict;
use Data::Dumper;
use LWP::UserAgent;
use base qw(Class::Accessor);
use CoGe::Accessory::GenBank::Feature;


__PACKAGE__->mk_accessors qw(id locus accn seq_length moltype division date definition verison keywords data_source dataset organism sequence srcfile dir anntoation features start stop chromosome add_gene_models);

sub new
  {
    my $self = shift;
    $self = __PACKAGE__->SUPER::new(@_);
    $self->features({}) unless $self->features;
    return $self;
  }

sub get_genbank_from_nbci
  {
    my $self = shift;
    my $id = shift;
    my $rev = shift;
    my $ua = new LWP::UserAgent;
    my $url = "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&qty=1&c_start=1&list_uids=$id&dopt=gb&send=Send&sendto=t&from=begin&to=end&extrafeatpresent=1&ef_MGC=16";
    my $search = $ua->get($url);
    unless ($search->is_success)
      {
	print STDERR "Trouble retrieving $id: ", $search->status_link,"\n";
      }

    return $self->parse_genbank($search->content, $rev);    

  }

sub parse_genbank
  {
    my $self = shift;
    my $gb = shift;
    my $rev = shift;
    my @gb = split /\n/, $gb;
    my %moltype_list = ( 'mRNA' => '1', 'RNA' => '1', 'tRNA' => '1',
			 'rRNA' => '1', 'DNA' => '1', 'scRNA' => '1',
			 'ds-mRNA' => '1', 'ss-DNA' => '1', 'snRNA' => '1'    );
    my($pos) = 0;
    my($locus,$length,$moltype,$division,$date) = "";
    my($accn,$definition,$version,$source,$organism) = "";
    my($keywords,$sequence) = "";
    my(@features) = ();

    unless ($gb[0] && $gb[0] =~ /^LOCUS/)
      {
	my $error =  "Doesn't appear to be a genbank file.  Returning. . .$gb\n";
	print STDERR $error;
	return (0,$error);
      }
    foreach my $line ( @gb ) 
      {
	# 1=locus, 2=length, 3=moltype, 4=form, 5=division, 6=date
	if ( $line =~ /^LOCUS\s+(\S+)\s+(\S+)\s+bp\s+(.*)/ ) 
	  {
	    $locus = $1; $length = $2;
	    my @temp = split( /\s+/, $3 );
	    if ( @temp >= 3 ) 
	      {
		if ( exists($moltype_list{ $temp[0] } ) ) 
		  {
		    $moltype = $temp[0];
		  }
		else 
		  {
		    $moltype = "?";
		  }
		$division = $temp[2];
		$date =  $temp[3] ;
	      }
	    elsif ( @temp == 2 ) 
	      {
		if ( exists($moltype_list{ $temp[0] } ) ) 
		  {
		    $moltype = $temp[0];
		    $division = "?";
		  }
		else 
		  {
		    $moltype = "?";
		    $division = $temp[0];
		  }
		$date = $temp[2] ;
	      }
	    else 
	      {
		warn "  GBFile::Locus regexp - Problems with input entry\n";
		return 0;
	      }
	  }
	elsif ( $line =~ /^DEFINITION\s+(.*)/ ) {
	  $definition .= $definition ? " $1" : $1;
	  if ($gb[ $pos+1 ] !~ /^ACCESSION/ ) {
	    $gb[ $pos+1 ] =
	      "DEFINITION" . $gb[ $pos+1 ];
	  }
	}
	elsif ( $line =~ /^ACCESSION\s+(\S+)/ ) {
	  $accn = $1;
	}
	elsif ( $line =~ /^VERSION\s+(.*)/ ) {
	  my @temp = split(/\s+/,$1);
	  $version = join(" ", @temp);
	}
	elsif ( $line =~ /^KEYWORDS\s+(.*)\./ ) {
	  $keywords = $1;
	}
	elsif ( $line =~ /^SOURCE\s+(.*)/ ) {
	  $source = $1;
	  $source =~ s/\.$//; #get rid of trailing ".", if present
	}
	elsif ( $line =~ /^\s+ORGANISM\s+(.*)/ ) {
	  $organism = $1;
	}
	elsif ( $line =~ /^FEATURES/ ) 
	  {
	    my $x = $pos+1;
	    my $iso;
	    my %feature;
	    while ( $pos < $x ) 
	      {
		if  ($gb[ $x ] =~ /^ORIGIN/)
		  {
		    my %tmp = %feature;
		    push @features, \%tmp if keys %tmp;
		    last;
		  }
		if ($gb[$x] =~ /^\s+(\S+)\s\s+(.*)$/)
		  {
		    my %tmp = %feature;
		    push @features, \%tmp if keys %tmp;
		    %feature = ();
		    $feature{type} = $1;
		    $iso = 'location';
		    $feature{$iso} = $2;
		  }
		elsif($gb[$x] =~ /\/(.*?)=(.*)$/)
		  {
		    $iso = $1;
		    $feature{$iso} = $2;
		  }
	        else
		  {
		    my $line = $gb[$x];
		    $line =~ s/\s+//g;
		    $feature{$iso} .= $line;
		  }
	        $x++;
	      }
	    }<commit_msg>added additional variable check<commit_after>package CoGe::Accessory::GenBank;

use strict;
use Data::Dumper;
use LWP::UserAgent;
use base qw(Class::Accessor);
use CoGe::Accessory::GenBank::Feature;


__PACKAGE__->mk_accessors qw(id locus accn seq_length moltype division date definition verison keywords data_source dataset organism sequence srcfile dir anntoation features start stop chromosome add_gene_models);

sub new
  {
    my $self = shift;
    $self = __PACKAGE__->SUPER::new(@_);
    $self->features({}) unless $self->features;
    return $self;
  }

sub get_genbank_from_nbci
  {
    my $self = shift;
    my $id = shift;
    my $rev = shift;
    my $ua = new LWP::UserAgent;
    my $url = "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&qty=1&c_start=1&list_uids=$id&dopt=gb&send=Send&sendto=t&from=begin&to=end&extrafeatpresent=1&ef_MGC=16";
    my $search = $ua->get($url);
    unless ($search->is_success)
      {
	print STDERR "Trouble retrieving $id: ", $search->status_link,"\n";
      }

    return $self->parse_genbank($search->content, $rev);    

  }

sub parse_genbank
  {
    my $self = shift;
    my $gb = shift;
    my $rev = shift;
    my @gb = split /\n/, $gb;
    my %moltype_list = ( 'mRNA' => '1', 'RNA' => '1', 'tRNA' => '1',
			 'rRNA' => '1', 'DNA' => '1', 'scRNA' => '1',
			 'ds-mRNA' => '1', 'ss-DNA' => '1', 'snRNA' => '1'    );
    my($pos) = 0;
    my($locus,$length,$moltype,$division,$date) = "";
    my($accn,$definition,$version,$source,$organism) = "";
    my($keywords,$sequence) = "";
    my(@features) = ();

    unless ($gb[0] && $gb[0] =~ /^LOCUS/)
      {
	my $error =  "Doesn't appear to be a genbank file.  Returning. . .$gb\n";
	print STDERR $error;
	return (0,$error);
      }
    foreach my $line ( @gb ) 
      {
	# 1=locus, 2=length, 3=moltype, 4=form, 5=division, 6=date
	if ( $line =~ /^LOCUS\s+(\S+)\s+(\S+)\s+bp\s+(.*)/ ) 
	  {
	    $locus = $1; $length = $2;
	    my @temp = split( /\s+/, $3 );
	    if ( @temp >= 3 ) 
	      {
		if ( exists($moltype_list{ $temp[0] } ) ) 
		  {
		    $moltype = $temp[0];
		  }
		else 
		  {
		    $moltype = "?";
		  }
		$division = $temp[2];
		$date =  $temp[3] ;
	      }
	    elsif ( @temp == 2 ) 
	      {
		if ( exists($moltype_list{ $temp[0] } ) ) 
		  {
		    $moltype = $temp[0];
		    $division = "?";
		  }
		else 
		  {
		    $moltype = "?";
		    $division = $temp[0];
		  }
		$date = $temp[2] ;
	      }
	    else 
	      {
		warn "  GBFile::Locus regexp - Problems with input entry\n";
		return 0;
	      }
	  }
	elsif ( $line =~ /^DEFINITION\s+(.*)/ ) {
	  $definition .= $definition ? " $1" : $1;
	  if ($gb[ $pos+1 ] !~ /^ACCESSION/ ) {
	    $gb[ $pos+1 ] =
	      "DEFINITION" . $gb[ $pos+1 ];
	  }
	}
	elsif ( $line =~ /^ACCESSION\s+(\S+)/ ) {
	  $accn = $1;
	}
	elsif ( $line =~ /^VERSION\s+(.*)/ ) {
	  my @temp = split(/\s+/,$1);
	  $version = join(" ", @temp);
	}
	elsif ( $line =~ /^KEYWORDS\s+(.*)\./ ) {
	  $keywords = $1;
	}
	elsif ( $line =~ /^SOURCE\s+(.*)/ ) {
	  $source = $1;
	  $source =~ s/\.$//; #get rid of trailing ".", if present
	}
	elsif ( $line =~ /^\s+ORGANISM\s+(.*)/ ) {
	  $organism = $1;
	}
	elsif ( $line =~ /^FEATURES/ ) 
	  {
	    my $x = $pos+1;
	    my $iso;
	    my %feature;
	    while ( $pos < $x ) 
	      {
		if  ($gb[ $x ] =~ /^ORIGIN/)
		  {
		    my %tmp = %feature;
		    push @features, \%tmp if keys %tmp;
		    last;
		  }
		if ($gb[$x] =~ /^\s+(\S+)\s\s+(.*)$/)
		  {
		    my %tmp = %feature;
		    push @features, \%tmp if keys %tmp;
		    %feature = ();
		    $feature{type} = $1;
		    $iso = 'location';
		    $feature{$iso} = $2;
		  }
		elsif($gb[$x] =~ /\/(.*?)=(.*)$/)
		  {
		    $iso = $1;
		    $feature{$iso} = $2;
		  }
	        else
		  {
		    my $line = $gb[$x];
		    $line =~ s/\s+//g;
		    $feature{$iso} .= $line if $iso;
		  }
	        $x++;
	      }
	    }
{'fraction_non_alphanumeric': 0.22567637877211238, 'fraction_numerical': 0.013787721123829344, 'mean_word_length': 2.211779448621554, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2227983', 'n_tokens_mistral': 3555, 'n_tokens_neox': 3143, 'n_words': 637}