<commit_before>#!/usr/local/bin/perl -w

=head1 NAME

  get_human_ests.pl

=head1 SYNOPSIS
 
  get_human_ests.pl

=head1 DESCRIPTION

  gets human ESTs from dbEST or cDNAs from embl/vertRNA and polyT/polyA clips them

=head1 OPTIONS

  -estfile
  -outfile

=cut


use strict; 
use Getopt::Long;
use Bio::Seq;
use Bio::SeqIO;
use Bio::EnsEMBL::Utils::PolyA;

#$| = 1; # disable buffering
#local $/ = '>';

my $estfile;
my $seqoutfile;
my $clip;
my $softmask;

############################################################
# we usually clip 20bp on either end of the EST to eliminate low quality sequence
my $clip_ends = 0;

############################################################
# we don't want any EST which is shorter than 60bp
my $min_length = 100;


&GetOptions( 
	    'estfile:s'     => \$estfile,
	    'outfile:s'     => \$seqoutfile,
	    'clip'          => \$clip,
	    'clip_ends:n'   => \$clip_ends,
	    'softmask'      => \$softmask,
	    'min_length'    => \$min_length,
	   );

# usage
if(!defined $estfile    ||
   !defined $seqoutfile 
  ){
  print STDERR "script to collect human ESTs.\n";
  print STDERR "It rejects ESTs labelled as pseudogenes, non-coding RNAs or cancer genes\n";
  print STDERR "\n";
  print STDERR "USAGE: get_human_ests.pl -estfile estfile -outfile outfile\n";
  print STDERR "                         -clip (clips polyA/T) -clip_ends n (clips n bases from both ends)\n";
  print STDERR "                                                             default = 0, 20 seems to be ok\n";
  print STDERR "                         -softmask ( softmask the polyA/T )\n";
  print STDERR "                         -min_length ( min_est_length, default = 60 )\n";
  exit(0);
}


my $seqin  = new Bio::SeqIO(-file   => "<$estfile",
			    -format => "Fasta",
			  );

my $seqout = new Bio::SeqIO(-file   => ">$seqoutfile", 
			    -format => "Fasta"
			   );

if ( $clip_ends ){
  print STDERR "clipping $clip_ends from both ends of ESTs\n";
}

SEQFETCH:
while( my $cdna = $seqin->next_seq ){<commit_msg>Correcting a documentation bug.<commit_after>#!/usr/local/bin/perl -w

=head1 NAME

  get_human_ests.pl

=head1 SYNOPSIS
 
  get_human_ests.pl

=head1 DESCRIPTION

  gets human ESTs from dbEST or cDNAs from embl/vertRNA and polyT/polyA clips them

=head1 OPTIONS

  -estfile
  -outfile

=cut


use strict; 
use Getopt::Long;
use Bio::Seq;
use Bio::SeqIO;
use Bio::EnsEMBL::Utils::PolyA;

#$| = 1; # disable buffering
#local $/ = '>';

my $estfile;
my $seqoutfile;
my $clip;
my $softmask;

############################################################
# we usually clip 20bp on either end of the EST to eliminate low quality sequence
my $clip_ends = 0;

############################################################
# we don't want any EST which is shorter than 100bp
my $min_length = 100;


&GetOptions( 
	    'estfile:s'     => \$estfile,
	    'outfile:s'     => \$seqoutfile,
	    'clip'          => \$clip,
	    'clip_ends:n'   => \$clip_ends,
	    'softmask'      => \$softmask,
	    'min_length'    => \$min_length,
	   );

# usage
if(!defined $estfile    ||
   !defined $seqoutfile 
  ){
  print STDERR "script to collect human ESTs.\n";
  print STDERR "It rejects ESTs labelled as pseudogenes, non-coding RNAs or cancer genes\n";
  print STDERR "\n";
  print STDERR "USAGE: get_human_ests.pl -estfile estfile -outfile outfile\n";
  print STDERR "                         -clip (clips polyA/T) -clip_ends n (clips n bases from both ends)\n";
  print STDERR "                                                             default = 0, 20 seems to be ok\n";
  print STDERR "                         -softmask ( softmask the polyA/T )\n";
  print STDERR "                         -min_length ( min_est_length, default = 100 )\n";
  exit(0);
}


my $seqin  = new Bio::SeqIO(-file   => "<$estfile",
			    -format => "Fasta",
			  );

my $seqout = new Bio::SeqIO(-file   => ">$seqoutfile", 
			    -format => "Fasta"
			   );

if ( $clip_ends ){
  print STDERR "clipping $clip_ends from both ends of ESTs\n";
}

SEQFETCH:
while( my $cdna = $seqin->next_seq ){
{'fraction_non_alphanumeric': 0.18059628543499512, 'fraction_numerical': 0.009775171065493646, 'mean_word_length': 2.5622280243690168, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2949263', 'n_tokens_mistral': 1532, 'n_tokens_neox': 1364, 'n_words': 397}