<commit_before>#!/usr/bin/env perl
#
# gtfSerializer.t
#
# Test Bio::EnsEMBL::Utils::IO::GTFSerializer.
#
# Strategy:
#
#   In theory, we should know the GTF output for some 
#   transcripts and compare it against the output of the 
#   serializer. In practice, this is difficult and very 
#   time consuming.
#   One way to circumnvent this is to start from the observation
#   that the serializer was built from the ashes of the web
#   script ensembl-webcode/utils/ensembl2gtf_forweb used by
#   production to get GTFs for last release (71).
#   The strategy is then to download the GTF files for all last release
#   species from the FTP site, and for each file compare its output
#   relative to transcripts with the corresponding output of the 
#   serializer.
#   
# Notes:
# 
#   - This strategy only works for testing the GTFSerializer during
#     the transition from 71 to 72, as after the current release the 
#     GTF files will be produced using the GTFSerializer itself via
#     the Hive pipeline.
#    
#   - The first run of the test reports 9 out 610 failuere for certain
#     species (canis_familiaris cavia_porcellus danio_rerio equus_caballus 
#     loxodonta_africana ornithorhynchus_anatinus oryctolagus_cuniculus 
#     xenopus_tropicalis). This is because the GTFSerializer has corrected
#     a bug of the original script, which was using an incorrect codon
#     table assumption in method check_start_and_stop which is not working
#     for all species.
#     Hence, this failures are ok.
#


use strict;
use warnings;

use Test::More;
use Test::Differences;
use IO::String;
use Net::FTP;

use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::ApiVersion qw/software_version/;
use Bio::EnsEMBL::Utils::IO qw/gz_work_with_file/; # iterate_lines
# use Bio::EnsEMBL::Test::MultiTestDB;
use Bio::EnsEMBL::Utils::Exception;
use Bio::EnsEMBL::Utils::IO::GTFSerializer;

# my $current_release = software_version();
# my $last_release = $current_release - 1;
my $release = 71;

my $registry = 'Bio::EnsEMBL::Registry';
eval { 
  $registry->load_registry_from_db(
    -host => 'mysql-ensembl-mirror.ebi.ac.uk',
    -port => 4240,
    -db_version => $release,
    -user => 'anonymous',
    -verbose => 0
  )
};
throw "Can't connect to db host" if $@;


SKIP: {
  note "Connecting to Ensembl FTP site";
  my $ftp = eval { ensembl_ftp_connect() };
  skip "Cannot login to Ensembl FTP site. We cannot continue with the tests", 1
    if $@;

  note "Retrieving species list";
  my $species_list = eval { get_species_list($ftp, $release) };<commit_msg>making the gtf serialiser test run only if you are the test author<commit_after>#!/usr/bin/env perl
#
# gtfSerializer.t
#
# Test Bio::EnsEMBL::Utils::IO::GTFSerializer.
#
# Strategy:
#
#   In theory, we should know the GTF output for some 
#   transcripts and compare it against the output of the 
#   serializer. In practice, this is difficult and very 
#   time consuming.
#   One way to circumnvent this is to start from the observation
#   that the serializer was built from the ashes of the web
#   script ensembl-webcode/utils/ensembl2gtf_forweb used by
#   production to get GTFs for last release (71).
#   The strategy is then to download the GTF files for all last release
#   species from the FTP site, and for each file compare its output
#   relative to transcripts with the corresponding output of the 
#   serializer.
#   
# Notes:
# 
#   - This strategy only works for testing the GTFSerializer during
#     the transition from 71 to 72, as after the current release the 
#     GTF files will be produced using the GTFSerializer itself via
#     the Hive pipeline.
#    
#   - The first run of the test reports 9 out 610 failuere for certain
#     species (canis_familiaris cavia_porcellus danio_rerio equus_caballus 
#     loxodonta_africana ornithorhynchus_anatinus oryctolagus_cuniculus 
#     xenopus_tropicalis). This is because the GTFSerializer has corrected
#     a bug of the original script, which was using an incorrect codon
#     table assumption in method check_start_and_stop which is not working
#     for all species.
#     Hence, this failures are ok.
#


use strict;
use warnings;

use Test::More;
use Test::Differences;
use IO::String;
use Net::FTP;

use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::ApiVersion qw/software_version/;
use Bio::EnsEMBL::Utils::IO qw/gz_work_with_file/; # iterate_lines
# use Bio::EnsEMBL::Test::MultiTestDB;
use Bio::EnsEMBL::Utils::Exception;
use Bio::EnsEMBL::Utils::IO::GTFSerializer;

if ( not $ENV{TEST_AUTHOR} ) {
  my $msg = 'Author test. Set $ENV{TEST_AUTHOR} to a true value to run.';
  plan( skip_all => $msg );
}

# my $current_release = software_version();
# my $last_release = $current_release - 1;
my $release = 71;

my $registry = 'Bio::EnsEMBL::Registry';
eval { 
  $registry->load_registry_from_db(
    -host => 'mysql-ensembl-mirror.ebi.ac.uk',
    -port => 4240,
    -db_version => $release,
    -user => 'anonymous',
    -verbose => 0
  )
};
throw "Can't connect to db host" if $@;


SKIP: {
  note "Connecting to Ensembl FTP site";
  my $ftp = eval { ensembl_ftp_connect() };
  skip "Cannot login to Ensembl FTP site. We cannot continue with the tests", 1
    if $@;

  note "Retrieving species list";
  my $species_list = eval { get_species_list($ftp, $release) };
{'fraction_non_alphanumeric': 0.09399133546807309, 'fraction_numerical': 0.007534375588623093, 'mean_word_length': 3.8940092165898617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3796385', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1715, 'n_words': 616}