<commit_before># Dump gene and xref information to an XML file for indexing by the EBI's
# search engine.
#
# To copy files to the EBI so that they can be picked up:
# scp homo_sapiens_core_41_36c.xml.gz <EMAIL>:xml/
#
# Email <EMAIL> after copying so the files can be indexed.

use strict;
use DBI;

use Getopt::Long;
use IO::Zlib;

use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Variation::DBSQL::DBAdaptor;

use HTML::Entities;

my ($host, $user, $pass, $port, $dbpattern, $max_genes, $gzip, $no_variation);

GetOptions( "host=s",              \$host,
	    "user=s",              \$user,
	    "pass=s",              \$pass,
	    "port=i",              \$port,
	    "dbpattern|pattern=s", \$dbpattern,
	    "gzip!",               \$gzip,
            "max_genes=i",         \$max_genes,
	    "no_variation",        \$no_variation,
	    "help" ,               \&usage
	  );

if( !$host || !$dbpattern ) {
  usage();
}

my $entry_count;

my $fh;

run();

sub run() {

  Bio::EnsEMBL::Registry->load_registry_from_db(-host => $host,
						-port => $port,
						-user => $user,
						-pass => $pass);

  # loop over databases

  my $dsn = "DBI:mysql:host=$host";
  $dsn .= ";port=$port" if ($port);

  my $db = DBI->connect( $dsn, $user, $pass );

  my @dbnames = map {$_->[0] } @{$db->selectall_arrayref("show databases")};

  for my $dbname (@dbnames) {

    next if ($dbname !~ /$dbpattern/);

    my $file = $dbname . ".xml";
    $file .= ".gz" if ($gzip);

    if ($gzip) {

      $fh = new IO::Zlib;
      $fh->open("$file", "wb9") || die ("Can't open compressed stream to $file");

    } else  {

      open(FILE, ">$file") || die "Can't open $file";

    }

    print "Dumping $dbname to $file\n";

    my $start_time = time;

    my $dba = new Bio::EnsEMBL::DBSQL::DBAdaptor('-host' => $host,
						 '-port' => $port,
						 '-user' => $user,
						 '-pass' => $pass,
						 '-dbname' => $dbname,
						 '-species' => $dbname);

    header($dba, $dbname);

    content($dba);

    footer();

    print_time($start_time);

  }

}

# -------------------------------------------------------------------------------

sub header {

  my ($dba, $dbname) = @_;

  p ("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>");
  p ("<!DOCTYPE database [ <!ENTITY auml \"&#228;\">]>");
  p ("<database>");
  p ("<name>$dbname</name>");

  my $meta_container = $dba->get_MetaContainer();
  my $species = @{$meta_container->list_value_by_key('species.common_name')}[0];

  p ("<description>Ensembl $species database</description>");

  my $release = @{$meta_container->list_value_by_key('schema_version')}[0];
  my $date = @{$meta_container->list_value_by_key('xref.timestamp')}[0]; # near enough for now
  $date = munge_release_date($date);

  p ("<release>$release</release>");
  p ("<release_date>$date</release_date>");

  p ("");
  p ("<entries>");


}

# -------------------------------------------------------------------------------

sub content {

  my ($dba) = @_;

  $entry_count = 0;

  my $gene_adaptor = $dba->get_GeneAdaptor();

  my $meta_container = $dba->get_MetaContainer();
  my $species = $meta_container->get_Species()->common_name();

  my $db_variation = variation_attach($dba) unless $no_variation;

  my $trv_adaptor;
  if ($db_variation) { # not all species have variation databases
    $trv_adaptor = $db_variation->get_TranscriptVariationAdaptor();
  }

  foreach my $gene (@{$gene_adaptor->fetch_all()}) {

    last if ($max_genes && $entry_count >= $max_genes);

    # general gene info
    p("");
    p ("<entry id=\"" . $gene->stable_id() . "\">");

    p ("<name>" . $gene->display_id() . "</name>");

    my $description = encode_entities($gene->description()); # do any other fields need encoding?

    p ("<description>" . $description . "</description>");

    my $created = $gene->created_date();
    my $modified = $gene->modified_date();

    if ($created || $modified) {
      p ("<dates>");
      p ("<date type=\"creation\" value=\"" . format_date($created) . "\"/>") if ($created);
      p ("<date type=\"last_modification\" value=\"" . format_date($modified) . "\"/>") if ($modified);
      p ("</dates>");
    }

    # xrefs - deal with protein_ids separately, as additional fields
    my @protein_ids;

    p ("<cross_references>");

    my $tax = $meta_container->get_taxonomy_id();
    p ("<ref dbname=\"taxonomy\" dbkey=\"$tax\"/>");

    foreach my $xref (@{$gene->get_all_DBLinks()}) {

      if ($xref->dbname() !~ /protein_id/) {
	my $display_id = $xref->display_id();
	$display_id =~ s/<.+>//g;
	p ("<ref dbname=\"" . $xref->dbname() ."\" dbkey=\"" . $display_id. "\"/>");
      } else {
	push @protein_ids, $xref->display_id();
      }
    }

    p ("</cross_references>");

    # additional fields - transcript, translation, species etc
    p ("<additional_fields>");

    my $transcripts = $gene->get_all_Transcripts();
    foreach my $transcript (@{$transcripts}) {

      p ("<field name=\"transcript\">" . $transcript->stable_id() . "</field>");

      my $translation = $transcript->translation();
      p ("<field name=\"translation\">" . $translation->stable_id() . "</field>") if ($translation);

    }

    foreach my $protein_id (@protein_ids) {
      p ("<field name=\"protein_id\">" . $protein_id . "</field>");
    }

    p ("<field name=\"species\">" . $species . "</field>");

    # SNP IDs
    if ($db_variation) {
      foreach my $tv (@{$trv_adaptor->fetch_all_by_Transcripts($transcripts)}){
	p ("<field name=\"variation_id\">" . $tv->variation_feature()->variation_name() . "</field>");
      }
    }

    p ("</additional_fields>");

    # close tag
    p ("</entry>");

    $entry_count++;

  }

}

# -------------------------------------------------------------------------------

sub footer {


  p ("</entries>");

  p ("<entry_count>$entry_count</entry_count>");<commit_msg>Use while/shift instead of foreach to reduce memory requirements.<commit_after># Dump gene and xref information to an XML file for indexing by the EBI's
# search engine.
#
# To copy files to the EBI so that they can be picked up:
# scp homo_sapiens_core_41_36c.xml.gz <EMAIL>:xml/
#
# Email <EMAIL> after copying so the files can be indexed.

use strict;
use DBI;

use Getopt::Long;
use IO::Zlib;

use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Variation::DBSQL::DBAdaptor;

use HTML::Entities;

my ($host, $user, $pass, $port, $dbpattern, $max_genes, $gzip, $no_variation);

GetOptions( "host=s",              \$host,
	    "user=s",              \$user,
	    "pass=s",              \$pass,
	    "port=i",              \$port,
	    "dbpattern|pattern=s", \$dbpattern,
	    "gzip!",               \$gzip,
            "max_genes=i",         \$max_genes,
	    "no_variation",        \$no_variation,
	    "help" ,               \&usage
	  );

if( !$host || !$dbpattern ) {
  usage();
}

my $entry_count;

my $fh;

run();

sub run() {

  Bio::EnsEMBL::Registry->load_registry_from_db(-host => $host,
						-port => $port,
						-user => $user,
						-pass => $pass);

  # loop over databases

  my $dsn = "DBI:mysql:host=$host";
  $dsn .= ";port=$port" if ($port);

  my $db = DBI->connect( $dsn, $user, $pass );

  my @dbnames = map {$_->[0] } @{$db->selectall_arrayref("show databases")};

  for my $dbname (@dbnames) {

    next if ($dbname !~ /$dbpattern/);

    my $file = $dbname . ".xml";
    $file .= ".gz" if ($gzip);

    if ($gzip) {

      $fh = new IO::Zlib;
      $fh->open("$file", "wb9") || die ("Can't open compressed stream to $file");

    } else  {

      open(FILE, ">$file") || die "Can't open $file";

    }

    print "Dumping $dbname to $file\n";

    my $start_time = time;

    my $dba = new Bio::EnsEMBL::DBSQL::DBAdaptor('-host' => $host,
						 '-port' => $port,
						 '-user' => $user,
						 '-pass' => $pass,
						 '-dbname' => $dbname,
						 '-species' => $dbname);

    header($dba, $dbname);

    content($dba);

    footer();

    print_time($start_time);

  }

}

# -------------------------------------------------------------------------------

sub header {

  my ($dba, $dbname) = @_;

  p ("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>");
  p ("<!DOCTYPE database [ <!ENTITY auml \"&#228;\">]>");
  p ("<database>");
  p ("<name>$dbname</name>");

  my $meta_container = $dba->get_MetaContainer();
  my $species = @{$meta_container->list_value_by_key('species.common_name')}[0];

  p ("<description>Ensembl $species database</description>");

  my $release = @{$meta_container->list_value_by_key('schema_version')}[0];
  my $date = @{$meta_container->list_value_by_key('xref.timestamp')}[0]; # near enough for now
  $date = munge_release_date($date);

  p ("<release>$release</release>");
  p ("<release_date>$date</release_date>");

  p ("");
  p ("<entries>");


}

# -------------------------------------------------------------------------------

sub content {

  my ($dba) = @_;

  $entry_count = 0;

  my $gene_adaptor = $dba->get_GeneAdaptor();

  my $meta_container = $dba->get_MetaContainer();
  my $species = $meta_container->get_Species()->common_name();

  my $db_variation = variation_attach($dba) unless $no_variation;

  my $trv_adaptor;
  if ($db_variation) { # not all species have variation databases
    $trv_adaptor = $db_variation->get_TranscriptVariationAdaptor();
  }

  my $genes = $gene_adaptor->fetch_all();

  while (my $gene = shift(@$genes)) {

    last if ($max_genes && $entry_count >= $max_genes);

    # general gene info
    p("");
    p ("<entry id=\"" . $gene->stable_id() . "\">");

    p ("<name>" . $gene->display_id() . "</name>");

    my $description = encode_entities($gene->description()); # do any other fields need encoding?

    p ("<description>" . $description . "</description>");

    my $created = $gene->created_date();
    my $modified = $gene->modified_date();

    if ($created || $modified) {
      p ("<dates>");
      p ("<date type=\"creation\" value=\"" . format_date($created) . "\"/>") if ($created);
      p ("<date type=\"last_modification\" value=\"" . format_date($modified) . "\"/>") if ($modified);
      p ("</dates>");
    }

    # xrefs - deal with protein_ids separately, as additional fields
    my @protein_ids;

    p ("<cross_references>");

    my $tax = $meta_container->get_taxonomy_id();
    p ("<ref dbname=\"taxonomy\" dbkey=\"$tax\"/>");

    my $xrefs = $gene->get_all_DBLinks();

    while (my $xref = shift(@$xrefs)) {

      if ($xref->dbname() !~ /protein_id/) {
	my $display_id = $xref->display_id();
	$display_id =~ s/<.+>//g;
	p ("<ref dbname=\"" . $xref->dbname() ."\" dbkey=\"" . $display_id. "\"/>");
      } else {
	push @protein_ids, $xref->display_id();
      }
    }

    p ("</cross_references>");

    # additional fields - transcript, translation, species etc
    p ("<additional_fields>");

    my $transcripts = $gene->get_all_Transcripts();

    foreach my $transcript (@$transcripts) { # can't use while/shift here as array needed later

      p ("<field name=\"transcript\">" . $transcript->stable_id() . "</field>");

      my $translation = $transcript->translation();
      p ("<field name=\"translation\">" . $translation->stable_id() . "</field>") if ($translation);

    }

    while (my $protein_id = shift(@protein_ids)) {
      p ("<field name=\"protein_id\">" . $protein_id . "</field>");
    }

    p ("<field name=\"species\">" . $species . "</field>");

    # SNP IDs
    if ($db_variation) {
      my $transcript_variations = $trv_adaptor->fetch_all_by_Transcripts($transcripts);
      while (my $tv = shift(@$transcript_variations)) {
	p ("<field name=\"variation_id\">" . $tv->variation_feature()->variation_name() . "</field>");
      }
    }

    p ("</additional_fields>");

    # close tag
    p ("</entry>");

    $entry_count++;

  }

}

# -------------------------------------------------------------------------------

sub footer {


  p ("</entries>");

  p ("<entry_count>$entry_count</entry_count>");
{'fraction_non_alphanumeric': 0.23938964395897608, 'fraction_numerical': 0.0033352789126990744, 'mean_word_length': 2.6159179981911365, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 2, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2126085', 'n_tokens_mistral': 4469, 'n_tokens_neox': 4059, 'n_words': 918}