<commit_before>#!/usr/local/bin/perl
# Copyright [2009-2014] EMBL-European Bioinformatics Institute
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#      http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Dump search XML files for indexing by the EBEye search engine.
#

package ebi_search_dump;

use strict;
use DBI;
use Carp;
use File::Find;
use Getopt::Long;
use IO::Zlib;
use Data::Dumper;
use HTML::Entities;
use FindBin qw($Bin);
use lib $Bin;
use LibDirs;
use utils::Tool;

my (
  $host,    $user,        $pass,   $port,     $species, $ind,
  $release, $max_entries, $nogzip, $parallel, $dir,     $inifile,
  $nogenetrees, $novariation, $noxrefs, $skip_existing, $format, $noortholog
);

my %rHash = map { $_ } @ARGV;
if ( $inifile = $rHash{'-inifile'} ) {
  my $icontent = `cat $inifile`;
  warn $icontent;
  eval $icontent;
}

GetOptions(
  "host=s",        \$host,        "port=i",    \$port,
  "user=s",        \$user,        "pass=s",    \$pass,
  "species=s",     \$species,     "release=s", \$release,
  "index=s",       \$ind,         "nogzip!",   \$nogzip,
  "max_entries=i", \$max_entries, "parallel",  \$parallel,
  "dir=s",         \$dir,         "help",      \&usage,
  "inifile=s",     \$inifile,  
  "nogenetrees",   \$nogenetrees,
  "novariation",   \$novariation,
  "noxrefs",       \$noxrefs,
  "skipexisting",   \$skip_existing,
  "format=s", \$format,
  "noortholog",       \$noortholog,
  );

$format  ||= 'ensembl';
$ind     ||= 'ALL';
$dir     ||= ".";
$release ||= 'LATEST';


if ($format eq 'solr') {
    $novariation = 1;
    $nogenetrees = 1;
    $nogzip = 1;
    $ind = 'Gene';
    $noortholog = 1;
}


usage() and exit unless ( $host && $port && $user);

## HACK 1 - if the INDEX is set to all grab all dumper methods...
my @indexes = split ',', $ind;
@indexes = map { /dump(\w+)/ ? $1 : () } keys %ebi_search_dump:: if $ind eq 'ALL';
#warn Dumper \@indexes;

my $dbHash = get_databases();
print "*** No databases found ***\n" unless %{$dbHash};
#warn Dumper $dbHash;

my @datasets = split ',', $species;
# restrict species to only those defined in the current eg site
@datasets = @datasets ? @{ utils::Tool::check_species(\@datasets) } : @{ utils::Tool::all_species() };

print "\nDatasets to process: \n  " . join("\n  ", @datasets) . "\n";

my $entry_count;
my $global_start_time = time;
my $total             = 0;
my $fh;

foreach my $dataset ( @datasets ) {
  my $conf = $dbHash->{lc($dataset)};

  foreach my $index (@indexes) {
    my $function = "dump$index";
    no strict "refs";

    $dataset =~ s/_/ /g;

    if ( $index eq 'Gene' ) {
      &$function( ucfirst($dataset), $conf );
    }
  }
}

print_time($global_start_time);
warn " Dumped $total entries ...\n";

#------------------------------------------------------------------------------

sub text_month {
  my $m = shift;
  my @months = qw[JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC];
  return $months[$m];
}

sub print_time {
  my $start = shift;
  my $t = time - $start;
  my $s = $t % 60;
  $t = ( $t - $s ) / 60;
  my $m = $t % 60;
  $t = ( $t - $m ) / 60;
  my $h = $t % 60;
  print "Time taken: " . $h . "h " . $m . "m " . $s . "s\n";
}

sub usage {
  print <<EOF; exit(0);

Usage: perl $0 <options>
  
  -host         REQUIRED. Database host to connect to. 
  -port         REQUIRED. Database port to connect to. 
  -user         Database username. Defaults to ensro.
  -species      Species name. Defaults to ALL.
  -index        Index to create. Defaults to ALL.
  -release      Release of the database to dump. Defaults to 'latest'.
  -pass         <PASSWORD>.
  -dir          Directory to write output to. Defaults to /lustre/scratch1/ensembl/gp1/xml.
  -nogzip       Don't compress output as it's written.
  -help         This message.
  -inifile      First take the arguments from this file. Then overwrite with what is provided in the command line

EOF
}

sub get_databases {

  my ( $dbHash, $dbcHash );
  
  my $dsn = "DBI:mysql:host=$host";
  $dsn .= ";port=$port" if ($port);

  my $dbh = DBI->connect( $dsn, $user, $pass );
  my @dbnames = map { $_->[0] } @{ $dbh->selectall_arrayref("show databases") };
  $dbh->disconnect();

  my $latest_release = 0;
  my ( $db_species, $db_release, $db_type );
  my $compara_hash;
  for my $dbname (@dbnames) {

    if ( ($db_type, $db_release) = $dbname =~ /^ensembl_compara_(\w+)_(\d+)_\w+/ ) {
       
      $compara_hash->{$db_type}->{$db_release} = $dbname;
    
    } elsif ( ( $db_species, $db_type, $db_release ) = $dbname =~ /^([a-z]+_[a-z0-9]+)(?:_collection)?_([a-z]+)_(\d+)_\w+$/ ) {

      $db_species =~ s/_collection$//;
      $latest_release = $db_release if ( $db_release > $latest_release );
      $dbHash->{$db_species}->{$db_type}->{$db_release} = $dbname;

    } 
  }

  map { $dbHash->{$_}->{'compara'} = $compara_hash } keys %$dbHash;
  $release = $latest_release if ( $release eq 'LATEST' );

  return $dbHash;
}

sub footer {
  my ($ecount) = @_;

  unless ($format eq 'solr') { 
  p("</entries>");
  p("<entry_count>$ecount</entry_count>");
  p("</database>");
}
  print "Dumped $ecount entries\n";
  if ($nogzip) {
    close(FILE) or die $!;<commit_msg>Modifications to search_dump.pl to work with trinomial species-bioproj database names<commit_after>#!/usr/local/bin/perl
# Copyright [2009-2014] EMBL-European Bioinformatics Institute
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#      http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Dump search XML files for indexing by the EBEye search engine.
#

package ebi_search_dump;

use strict;
use DBI;
use Carp;
use File::Find;
use Getopt::Long;
use IO::Zlib;
use Data::Dumper;
use HTML::Entities;
use FindBin qw($Bin);
use lib $Bin;
use LibDirs;
use utils::Tool;

my (
  $host,    $user,        $pass,   $port,     $species, $ind,
  $release, $max_entries, $nogzip, $parallel, $dir,     $inifile,
  $nogenetrees, $novariation, $noxrefs, $skip_existing, $format, $noortholog
);

my %rHash = map { $_ } @ARGV;
if ( $inifile = $rHash{'-inifile'} ) {
  my $icontent = `cat $inifile`;
  warn $icontent;
  eval $icontent;
}

GetOptions(
  "host=s",        \$host,        "port=i",    \$port,
  "user=s",        \$user,        "pass=s",    \$pass,
  "species=s",     \$species,     "release=s", \$release,
  "index=s",       \$ind,         "nogzip!",   \$nogzip,
  "max_entries=i", \$max_entries, "parallel",  \$parallel,
  "dir=s",         \$dir,         "help",      \&usage,
  "inifile=s",     \$inifile,  
  "nogenetrees",   \$nogenetrees,
  "novariation",   \$novariation,
  "noxrefs",       \$noxrefs,
  "skipexisting",   \$skip_existing,
  "format=s", \$format,
  "noortholog",       \$noortholog,
  );

$format  ||= 'ensembl';
$ind     ||= 'ALL';
$dir     ||= ".";
$release ||= 'LATEST';


if ($format eq 'solr') {
    $novariation = 1;
    $nogenetrees = 1;
    $nogzip = 1;
    $ind = 'Gene';
    $noortholog = 1;
}


usage() and exit unless ( $host && $port && $user);

## HACK 1 - if the INDEX is set to all grab all dumper methods...
my @indexes = split ',', $ind;
@indexes = map { /dump(\w+)/ ? $1 : () } keys %ebi_search_dump:: if $ind eq 'ALL';
#warn Dumper \@indexes;

my $dbHash = get_databases();
print "*** No databases found ***\n" unless %{$dbHash};
#warn Dumper $dbHash;

my @datasets = split ',', $species;
# restrict species to only those defined in the current eg site
@datasets = @datasets ? @{ utils::Tool::check_species(\@datasets) } : @{ utils::Tool::all_species() };

print "\nDatasets to process: \n  " . join("\n  ", @datasets) . "\n";

my $entry_count;
my $global_start_time = time;
my $total             = 0;
my $fh;

foreach my $dataset ( @datasets ) {
  my $conf = $dbHash->{lc($dataset)};

  foreach my $index (@indexes) {
    my $function = "dump$index";
    no strict "refs";

    $dataset =~ s/_/ /g;

    if ( $index eq 'Gene' ) {
      &$function( ucfirst($dataset), $conf );
    }
  }
}

print_time($global_start_time);
warn " Dumped $total entries ...\n";

#------------------------------------------------------------------------------

sub text_month {
  my $m = shift;
  my @months = qw[JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC];
  return $months[$m];
}

sub print_time {
  my $start = shift;
  my $t = time - $start;
  my $s = $t % 60;
  $t = ( $t - $s ) / 60;
  my $m = $t % 60;
  $t = ( $t - $m ) / 60;
  my $h = $t % 60;
  print "Time taken: " . $h . "h " . $m . "m " . $s . "s\n";
}

sub usage {
  print <<EOF; exit(0);

Usage: perl $0 <options>
  
  -host         REQUIRED. Database host to connect to. 
  -port         REQUIRED. Database port to connect to. 
  -user         Database username. Defaults to ensro.
  -species      Species name. Defaults to ALL.
  -index        Index to create. Defaults to ALL.
  -release      Release of the database to dump. Defaults to 'latest'.
  -pass         Password for <PASSWORD>.
  -dir          Directory to write output to. Defaults to /lustre/scratch1/ensembl/gp1/xml.
  -nogzip       Don't compress output as it's written.
  -help         This message.
  -inifile      First take the arguments from this file. Then overwrite with what is provided in the command line

EOF
}

sub get_databases {

  my ( $dbHash, $dbcHash );
  
  my $dsn = "DBI:mysql:host=$host";
  $dsn .= ";port=$port" if ($port);

  my $dbh = DBI->connect( $dsn, $user, $pass );
  my @dbnames = map { $_->[0] } @{ $dbh->selectall_arrayref("show databases") };
  $dbh->disconnect();

  my $latest_release = 0;
  my ( $db_species, $db_release, $db_type );
  my $compara_hash;
  for my $dbname (@dbnames) {

    if ( ($db_type, $db_release) = $dbname =~ /^ensembl_compara_(\w+)_(\d+)_\w+/ ) {
       
      $compara_hash->{$db_type}->{$db_release} = $dbname;

    } elsif ( ( $db_species, $db_type, $db_release ) = $dbname =~ /^([a-z]+_[a-z0-9]+_[a-z0-9]+)(?:_collection)?_([a-z]+)_(\d+)_\w+$/ ) {    

      $db_species =~ s/_collection$//;
      $latest_release = $db_release if ( $db_release > $latest_release );
      $dbHash->{$db_species}->{$db_type}->{$db_release} = $dbname;

    } 
  }

  map { $dbHash->{$_}->{'compara'} = $compara_hash } keys %$dbHash;
  $release = $latest_release if ( $release eq 'LATEST' );

  return $dbHash;
}

sub footer {
  my ($ecount) = @_;

  unless ($format eq 'solr') { 
  p("</entries>");
  p("<entry_count>$ecount</entry_count>");
  p("</database>");
}
  print "Dumped $ecount entries\n";
  if ($nogzip) {
    close(FILE) or die $!;
{'fraction_non_alphanumeric': 0.1848702276552396, 'fraction_numerical': 0.0067322172025865885, 'mean_word_length': 2.770875083500334, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6847261', 'n_tokens_mistral': 4336, 'n_tokens_neox': 4138, 'n_words': 1173}