<commit_before>
=head1 NAME

  Bio::EnsEMBL::Analysis::Tools::S3Utils - base class for data retrieval from S3 storage 


=head1 SYNOPSIS

  Bio::EnsEMBL::Analysis::Tools::S3Utils qw(get_file_from_s3) 


  Bio::EnsEMBL::Analysis::Tools::S3Utils 

  to import all methods 

=head1 DESCRIPTION

This is a base class for Utility modules for data retreival from S3. 
The module is wrappign the s3cmd command. This command requires a 
config file which can be created with the s3cmd --configure option 

=head1 CONTACT

please send any questions to <EMAIL>

=head1 METHODS 

the rest of the documention details the exported static class methods

=cut


package Bio::EnsEMBL::Analysis::Tools::S3Utils; 

use strict;
use warnings;
use Exporter;
use Bio::EnsEMBL::Utils::Exception qw(verbose throw warning stack_trace_dump); 


use Bio::EnsEMBL::Analysis::Config::S3Config; 
use Bio::EnsEMBL::Analysis::Config::General; 

use vars qw (@ISA  @EXPORT);

@ISA = qw(Exporter);
@EXPORT = qw(get_file_from_s3) ; 


sub get_file_from_s3 { 
  my ( $s3_bucket,$s3_file,$local_file ,$s3_config_file, $check) = @_ ;  

  if (!defined $s3_config_file ) { 
    $s3_config_file = $S3_CONFIG_FILE; 
  }  
  check_config_file ($s3_config_file); 

  my $cmd = "s3cmd -c $s3_config_file get $s3_bucket/$s3_file $local_file";
  system($cmd);  

  if ( $check ==1 ) {    
     my @lines = @{ get_md5sum($s3_bucket,$s3_file,$s3_config_file) } ;  
     for ( @lines ) {  
       print $_ . "\n"; 
     } 
     my $cmd = "s3cmd -c $s3_config_file ls --list-md5sum get $s3_bucket/$s3_file $md5sum_file";
    
  } 
}


sub get_md5sum {  
  my ( $s3_bucket,$s3_file,$s3_config_file ) = @_ ;  

  my $tmp_file;  
  my $cmd = "s3cmd -c $s3_config_file ls --list-md5 $s3_bucket/$s3_file > $tmp_file "; 
  local *SCMD;
  open(SCMD, "$cmd 2>&1 |") or throw("couldn't open pipe s3cmd");
  while(my $line = <SCMD>){ 
    push @all, $line ; 
  } 
  return \@all; 
} 

<commit_msg>methods for retrieval of gz files , uncompression and md5sum checks<commit_after>
=head1 NAME

  Bio::EnsEMBL::Analysis::Tools::S3Utils - base class for data retrieval from S3 storage 


=head1 SYNOPSIS

  Bio::EnsEMBL::Analysis::Tools::S3Utils qw(get_file_from_s3) 


  Bio::EnsEMBL::Analysis::Tools::S3Utils 

  to import all methods 

=head1 DESCRIPTION

This is a base class for Utility modules for data retreival from S3. 
The module is wrappign the s3cmd command. This command requires a 
config file which can be created with the s3cmd --configure option 

=head1 CONTACT

please send any questions to <EMAIL>

=head1 METHODS 

the rest of the documention details the exported static class methods

=cut


package Bio::EnsEMBL::Analysis::Tools::S3Utils; 

use strict;
use warnings;
use Exporter;
use Bio::EnsEMBL::Utils::Exception qw(verbose throw warning stack_trace_dump); 


use Bio::EnsEMBL::Analysis::Config::S3Config; 
use Bio::EnsEMBL::Analysis::Config::General; 
use Digest::MD5; 
use IO::Uncompress::Gunzip qw(gunzip $GunzipError) ; 
use vars qw (@ISA  @EXPORT);

@ISA = qw(Exporter);
@EXPORT = qw(
             get_file_from_s3
             get_file_from_s3_and_gunzip
            ) ; 



# mini routine to overwrite config vars. if direct value passed in, direct value used, 
# if not standard val in S3Conf used, if set no config ( ie $HOME/.s3cfg ) will be used 

sub s3_conf_para {   
  my ( $s3_config_file ) = @_; 
  my $s3_conf; 
  if (defined $s3_config_file ) { 
      $s3_conf = $s3_config_file; 
  }elsif ( defined $S3_CONFIG_FILE ) {  
      $s3_conf =$S3_CONFIG_FILE;
  }  
  my $use_config = "";
  if ( defined $s3_conf ) { 
    check_config_file($s3_conf);  
    $use_config = " -c $s3_conf ";
  } 
  return $use_config;
} 



sub get_file_from_s3_and_gunzip { 
  my ( $s3_bucket,$s3_file,$local_dir ,$check, $s3_config_file ) = @_ ; 

  my $lf =  get_file_from_s3($s3_bucket,$s3_file,$local_dir ,$check, $s3_config_file);  
  my $lfz = $lf."gz";
  system("mv $lf $lfz");
  my $outf = $lf.".$$.gunzip"; 
   
  gunzip $lfz=> $outf or throw(" Gunzip failed : $GunzipError");
  return $outf; 
}


sub get_file_from_s3 { 
  my ( $s3_bucket,$s3_file,$local_dir ,$check, $s3_config_file ) = @_ ;  

  my $use_config = s3_conf_para($s3_config_file); 

  my $local_file = $local_dir ."/".$s3_file.".$$";
  my $cmd = "s3cmd --force $use_config get $s3_bucket/$s3_file $local_file"; 
  system($cmd);  

  if (defined $check && $check ==1 ) {     
     my $md5;
     my @lines = @{ get_md5sum($s3_bucket,$s3_file,$s3_config_file) } ;  
     for my $l ( @lines ) {  
       my @item = split /\s+/,$l; 
       if ( $item[4]=~m/$s3_file/) { 
         $md5=$item[3]; 
       }
     }  
     if ( check_file_vs_md5 ($md5,$local_file) ) {  
       return $local_file; 
     } else {  
       throw("md5sums for downloded file and local file do not match \n"); 
     } 
  }  
  return $local_file;
}



sub check_file_vs_md5{ 
  my ( $md5,$local_file) = @_  ;  

  my $ctx = Digest::MD5->new;
  open(F,$local_file); 
  $ctx->addfile(*F); 
  my $ms = $ctx->hexdigest;   
  if ( $ms eq $md5 ){  
    return 1;
  } 
  return 0 ; 
} 

sub get_md5sum {  
  my ( $s3_bucket,$s3_file,$s3_config_file ) = @_ ;   

  my $use_config = s3_conf_para($s3_config_file); 

  my @all;
  my $cmd = "s3cmd $use_config ls --list-md5 $s3_bucket/$s3_file"; 
  local *SCMD;
  open(SCMD, "$cmd 2>&1 |") or throw("couldn't open pipe s3cmd");
  while(my $line = <SCMD>){ 
    push @all, $line ; 
  } 
  return \@all; 
} 


{'fraction_non_alphanumeric': 0.13307635829662262, 'fraction_numerical': 0.02459618208516887, 'mean_word_length': 3.3837489943684633, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2970490', 'n_tokens_mistral': 2280, 'n_tokens_neox': 2119, 'n_words': 509}