<commit_before>=head1 LICENSE
Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2019] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=cut

=head1 CONTACT
Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.
Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.
=cut
package Bio::EnsEMBL::Variation::Pipeline::ReleaseDataDumps::InitSubmitJob;

use strict;
use warnings;
use FileHandle;
use Data::Dumper;
use Bio::EnsEMBL::Utils::Slice qw(split_Slices);
use base ('Bio::EnsEMBL::Variation::Pipeline::ReleaseDataDumps::BaseDataDumpsProcess');

my $global_vf_count_in_species; #= 5_000_000; # if number of vf in a species exceeds this we need to split up dumps
my $max_vf_load; #= 2_000_000; # group slices together until the vf count exceeds max_vf_load
my $vf_per_slice; #= 2_000_000; # if number of vf exceeds this we split the slice and dump for each split slice
my $max_split_slice_length; #= 500_000;

my $overlap = 1;
my $debug = 0;
my $debug_fh;

sub fetch_input {
  my $self = shift;
  my $species   = $self->param('species');
  my $config    = $self->param('config');
  my $release   = $self->param('release');
  my $assembly = $self->get_assembly;

  my $job_type  = $self->param('job_type'); # parse or dump

  $debug = $self->param('debug');

  my $output_dir = $self->data_dir($species);
  if ($job_type eq 'dump') {
    # Create directory for species which will contain the species specific GVF dumps
    my $gvf_dir = "$output_dir/gvf/$species/";
    $self->create_species_dir($gvf_dir);
  }

  $global_vf_count_in_species = $self->param('global_vf_count_in_species') || $global_vf_count_in_species;
  $max_vf_load = $self->param('max_vf_load') || $max_vf_load; # group slices together until the vf count exceeds max_vf_load
  $vf_per_slice = $self->param('vf_per_slice') || $vf_per_slice; # if number of vf exceeds this we split the slice and dump for each split slice
  $max_split_slice_length = $self->param('max_split_slice_length') || $max_split_slice_length;

  if ($debug) {
    $debug_fh =  FileHandle->new("$output_dir/$species\_initSubmitJob.txt", 'w');
  }
  my $script_args_to_file_name = {};
  my $input;
  foreach my $dump_type (keys %$config) { # generic, sets, incl_consequences, svs
  # config could look like this:
  # "config" => {"failed" => ["failed"],"generic" => ["evidence"],"incl_consequences" => ["incl_consequences","protein_coding_details","evidence"]
    # We only dump failed variants as GVF files. We don't convert those files to VCF and skip them here
    if ($dump_type eq 'failed') {
      next if ($job_type eq 'parse');
    }
    # We dump two sets (clinically and phenotype associated) for human. We need to tell the dump_gvf script that we want
    # to dump a set from the variation datbase by providing the set name for example --set_name clinically_associated
    if ($dump_type eq 'sets') {
      foreach my $set_name (keys %{$config->{sets}}) {
        my @arguments = map {'--' . $_} @{$config->{sets}->{$set_name}};
        my $script_arg = join(' ', @arguments);
        $script_arg = "--set_name $set_name $script_arg";
        my $file_name = "$species\_$set_name";
        $script_args_to_file_name->{$script_arg} = $file_name;
      }
    } else {
      my @arguments = map {'--' . $_} @{$config->{$dump_type}};
      my $script_arg = join(' ', @arguments);
      my $file_name = "$species\_$dump_type";
      $script_args_to_file_name->{$script_arg} = $file_name;
    }
  }
  # script_args_to_file_name looks like this for macaca_mulatta
# {
# '--failed' => 'macaca_mulatta_failed',
# '--incl_consequences --protein_coding_details --evidence --ancestral_allele' => 'macaca_mulatta_incl_consequences',
# '--evidence --ancestral_allele' => 'macaca_mulatta_generic',
# '--structural_variations' => 'macaca_mulatta_structural_variations'
# };

  if ($job_type eq 'dump') {  
    my $global_vf_count = $self->get_global_vf_count();
    if ($debug) {
      print $debug_fh "GLOBAL_VF_COUNT\t$global_vf_count\n";
    }
    if ($global_vf_count > $global_vf_count_in_species) {<commit_msg>exclude script args for gvf2vcf script<commit_after>=head1 LICENSE
Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2019] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=cut

=head1 CONTACT
Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.
Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.
=cut
package Bio::EnsEMBL::Variation::Pipeline::ReleaseDataDumps::InitSubmitJob;

use strict;
use warnings;
use FileHandle;
use Data::Dumper;
use Bio::EnsEMBL::Utils::Slice qw(split_Slices);
use base ('Bio::EnsEMBL::Variation::Pipeline::ReleaseDataDumps::BaseDataDumpsProcess');

my $global_vf_count_in_species; #= 5_000_000; # if number of vf in a species exceeds this we need to split up dumps
my $max_vf_load; #= 2_000_000; # group slices together until the vf count exceeds max_vf_load
my $vf_per_slice; #= 2_000_000; # if number of vf exceeds this we split the slice and dump for each split slice
my $max_split_slice_length; #= 500_000;

my $overlap = 1;
my $debug = 0;
my $debug_fh;

sub fetch_input {
  my $self = shift;
  my $species   = $self->param('species');
  my $config    = $self->param('config');
  my $release   = $self->param('release');
  my $assembly = $self->get_assembly;

  my $job_type  = $self->param('job_type'); # parse or dump

  $debug = $self->param('debug');

  my $output_dir = $self->data_dir($species);
  if ($job_type eq 'dump') {
    # Create directory for species which will contain the species specific GVF dumps
    my $gvf_dir = "$output_dir/gvf/$species/";
    $self->create_species_dir($gvf_dir);
  }

  $global_vf_count_in_species = $self->param('global_vf_count_in_species') || $global_vf_count_in_species;
  $max_vf_load = $self->param('max_vf_load') || $max_vf_load; # group slices together until the vf count exceeds max_vf_load
  $vf_per_slice = $self->param('vf_per_slice') || $vf_per_slice; # if number of vf exceeds this we split the slice and dump for each split slice
  $max_split_slice_length = $self->param('max_split_slice_length') || $max_split_slice_length;

  if ($debug) {
    $debug_fh =  FileHandle->new("$output_dir/$species\_initSubmitJob.txt", 'w');
  }
  my $script_args_to_file_name = {};
  my $input;
  foreach my $dump_type (keys %$config) { # generic, sets, incl_consequences, svs
  # config could look like this:
  # "config" => {"failed" => ["failed"],"generic" => ["evidence"],"incl_consequences" => ["incl_consequences","protein_coding_details","evidence"]
    # We only dump failed variants as GVF files. We don't convert those files to VCF and skip them here
    if ($dump_type eq 'failed') {
      next if ($job_type eq 'parse');
    }
    # We dump two sets (clinically and phenotype associated) for human. We need to tell the dump_gvf script that we want
    # to dump a set from the variation datbase by providing the set name for example --set_name clinically_associated
    if ($dump_type eq 'sets') {
      foreach my $set_name (keys %{$config->{sets}}) {
        next if ($job_type eq 'parse'); # we don't need to pass --set_name to gvf2vcf.pl
        my @arguments = map {'--' . $_} @{$config->{sets}->{$set_name}};
        my $script_arg = join(' ', @arguments);
        $script_arg = "--set_name $set_name $script_arg";
        my $file_name = "$species\_$set_name";
        $script_args_to_file_name->{$script_arg} = $file_name;
      }
    } else {
      my @arguments = ();
      foreach my $argument (@{$config->{$dump_type}}) {
        next if ($argument eq 'somatic' && ($job_type eq 'parse')); # we don't need tp pass --somatic to gvf2vcf.pl 
        push @arguments, "--$argument";
      }
      my $script_arg = join(' ', @arguments);
      my $file_name = "$species\_$dump_type";
      $script_args_to_file_name->{$script_arg} = $file_name;
    }
  }
  # script_args_to_file_name looks like this for macaca_mulatta
# {
# '--failed' => 'macaca_mulatta_failed',
# '--incl_consequences --protein_coding_details --evidence --ancestral_allele' => 'macaca_mulatta_incl_consequences',
# '--evidence --ancestral_allele' => 'macaca_mulatta_generic',
# '--structural_variations' => 'macaca_mulatta_structural_variations'
# };

  if ($job_type eq 'dump') {  
    my $global_vf_count = $self->get_global_vf_count();
    if ($debug) {
      print $debug_fh "GLOBAL_VF_COUNT\t$global_vf_count\n";
    }
    if ($global_vf_count > $global_vf_count_in_species) {
{'fraction_non_alphanumeric': 0.12565230737746855, 'fraction_numerical': 0.010743886217128825, 'mean_word_length': 4.40896513558384, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4334547', 'n_tokens_mistral': 3338, 'n_tokens_neox': 3140, 'n_words': 943}