<commit_before>#!/usr/bin/env perl
use strict;
use warnings;
use ReseqTrack::Tools::Exception;
use ReseqTrack::DBSQL::DBAdaptor;
use File::Basename;
use Getopt::Long;
use ReseqTrack::File;
use ReseqTrack::Host;
use autodie;

my $dbhost;
my $dbuser;
my $dbpass;
my $dbport;
my $dbname;
my $run_metadata_file;
my $exp_metadata_file;
my $run_keyword        = 'RUN_ID';
my $exp_keyword        = 'EXPERIMENT_ID';
my $type_list_file;
my @type_list;
my $aln_dir;
my $result_dir;
my $vcf_dir;
my $species            = 'homo_sapiens';
my $genome_version     = 'GRCh38';

GetOptions(
        'dbhost=s'            => \$dbhost,
        'dbname=s'            => \$dbname,
        'dbuser=s'            => \$dbuser,
        'dbpass=s'            => \$dbpass,
        'dbport=s'            => \$dbport,
        'run_metadata_file=s' => \$run_metadata_file,
        'exp_metadata_file=s' => \$exp_metadata_file,
        'run_keyword=s'       => \$run_keyword,
        'exp_keyword=s'       => \$exp_keyword,
        'type_list_file=s'    => \$type_list_file,
        'type_list=s'         => \@type_list,
        'aln_dir=s'           => \$aln_dir,
        'vcf_dir=s'           => \$vcf_dir,
        'result_dir=s'        => \$result_dir,
        'species=s'           => \$species,
        'genome_version=s'    => \$genome_version,
      );

die `perldoc -t $0` if ( !$run_metadata_file && !$exp_metadata_file ) || 
                       ( !$run_keyword && !$exp_keyword ) || 
                       ( !$type_list_file && @type_list == 0 ) ||
                       !$aln_dir || !$vcf_dir || !$result_dir || 
                       !$species || !$genome_version;

my %file_path_hash = ( aln_dir        => $aln_dir,
                       vcf_dir        => $vcf_dir,
                       result_dir     => $result_dir,
                       species        => $species,
                       genome_version => $genome_version,
                     );

my $db = ReseqTrack::DBSQL::DBAdaptor->new(
          -host   => $dbhost,
          -user   => $dbuser,
          -port   => $dbport,
          -dbname => $dbname,
          -pass   => $dbpass,
        );

my $ca = $db->get_CollectionAdaptor;
my $fa = $db->get_FileAdaptor;
my $ha = $db->get_HostAdaptor;

my $file_types    = defined $type_list_file ? 
                    get_list( $type_list_file ) : \@type_list;

my $meta_hash = defined $run_metadata_file ? 
                get_index_hash( $run_metadata_file, $run_keyword ) : 
                get_index_hash( $exp_metadata_file, $exp_keyword );

TYPE:
foreach my $type ( @$file_types ){ 
  my $cs = $ca->fetch_by_type( $type );
  next TYPE if !$cs;

  COLLECTION:
  foreach my $c ( @{$cs} ){
    next COLLECTION if $c->table_name ne 'file';
    my $collection_name = $c->name;

    for my $f ( @{ $c->others } ) {
      my $file_path = $f->name;
      my $file_size = $f->size;
      my $file_type = $f->type;
      my $file_md5  = $f->md5;
      my $host_id   = $f->host_id;
      
      next COLLECTION if $f->withdrawn eq 1;        ## not moving withdrawn files

      my $host = $ha->fetch_by_dbID($host_id);
      next COLLECTION if $host->remote eq 1;        ## not moving remote files

      my $new_path;
  
      my %new_path_hash = ( path_hash        => \%file_path_hash, 
                            metadata         => $meta_hash,
                            collection_name  => $collection_name,
                            file_path        => $file_path,
                            type             => $type,
                          );

      if ( $collection_name =~ /^ERR\d+/ ){         ## collections with ERR ids
        die "Run metadata file not defined"
           unless $run_metadata_file;
 
        $new_path_hash{ change_name } = 0;          ## keeping existing file name for run types
        $new_path = get_new_path( \%new_path_hash );
      }
      elsif ( $collection_name =~ /^ERX\d+/ ){      ## collections with ERX ids
        die "Exp metadata file not defined"
           unless $exp_metadata_file;

        $new_path_hash{ change_name } = 1;          ## changing file name for run types
        $new_path = get_new_path( \%new_path_hash );
      }
      throw("check collection name: $collection_name ,type: $file_type ,file: $file_path") if !$new_path;

      print join("\t", $file_path, $new_path),$/
          if $file_path ne $new_path;
    }
  }
}


sub get_list {
  my ( $file ) = @_;
  my @list;

  open my $fh, '<', $file;
  while( <$fh> ){
    chomp;
    push @list,$_;
  } 
  return \@list;
}

sub get_index_hash {
my ( $file, $key_string ) = @_;
  open my $fh, '<', $file;
  my @header;
  my %data;
  my $key_index = undef;

  while ( <$fh> ) {
    chomp;
    next if m/^#/;
    my @vals = split "\t", $_;

    if ( @header ) {
      die "$key_string not found in $file\n" unless $key_index >= 0;
      $data { $vals[$key_index] }{ $header[$_] } = $vals[$_] for 0..$#header;
    }
    else {
      @header = map { uc($_) } @vals;
      my @key_index_array = grep{ $header[$_] eq $key_string } 0..$#header;
      $key_index = $key_index_array[0];
    }<commit_msg>added check for missing keyword<commit_after>#!/usr/bin/env perl
use strict;
use warnings;
use ReseqTrack::Tools::Exception;
use ReseqTrack::DBSQL::DBAdaptor;
use File::Basename;
use Getopt::Long;
use ReseqTrack::File;
use ReseqTrack::Host;
use autodie;

my $dbhost;
my $dbuser;
my $dbpass;
my $dbport;
my $dbname;
my $run_metadata_file;
my $exp_metadata_file;
my $run_keyword        = 'RUN_ID';
my $exp_keyword        = 'EXPERIMENT_ID';
my $type_list_file;
my @type_list;
my $aln_dir;
my $result_dir;
my $vcf_dir;
my $species            = 'homo_sapiens';
my $genome_version     = 'GRCh38';

GetOptions(
        'dbhost=s'            => \$dbhost,
        'dbname=s'            => \$dbname,
        'dbuser=s'            => \$dbuser,
        'dbpass=s'            => \$dbpass,
        'dbport=s'            => \$dbport,
        'run_metadata_file=s' => \$run_metadata_file,
        'exp_metadata_file=s' => \$exp_metadata_file,
        'run_keyword=s'       => \$run_keyword,
        'exp_keyword=s'       => \$exp_keyword,
        'type_list_file=s'    => \$type_list_file,
        'type_list=s'         => \@type_list,
        'aln_dir=s'           => \$aln_dir,
        'vcf_dir=s'           => \$vcf_dir,
        'result_dir=s'        => \$result_dir,
        'species=s'           => \$species,
        'genome_version=s'    => \$genome_version,
      );

die `perldoc -t $0` if ( !$run_metadata_file && !$exp_metadata_file ) || 
                       ( !$run_keyword && !$exp_keyword ) || 
                       ( !$type_list_file && @type_list == 0 ) ||
                       !$aln_dir || !$vcf_dir || !$result_dir || 
                       !$species || !$genome_version;

my %file_path_hash = ( aln_dir        => $aln_dir,
                       vcf_dir        => $vcf_dir,
                       result_dir     => $result_dir,
                       species        => $species,
                       genome_version => $genome_version,
                     );

my $db = ReseqTrack::DBSQL::DBAdaptor->new(
          -host   => $dbhost,
          -user   => $dbuser,
          -port   => $dbport,
          -dbname => $dbname,
          -pass   => $<PASSWORD>,
        );

my $ca = $db->get_CollectionAdaptor;
my $fa = $db->get_FileAdaptor;
my $ha = $db->get_HostAdaptor;

my $file_types    = defined $type_list_file ? 
                    get_list( $type_list_file ) : \@type_list;

my $meta_hash = defined $run_metadata_file ? 
                get_index_hash( $run_metadata_file, $run_keyword ) : 
                get_index_hash( $exp_metadata_file, $exp_keyword );

TYPE:
foreach my $type ( @$file_types ){ 
  my $cs = $ca->fetch_by_type( $type );
  next TYPE if !$cs;

  COLLECTION:
  foreach my $c ( @{$cs} ){
    next COLLECTION if $c->table_name ne 'file';
    my $collection_name = $c->name;

    for my $f ( @{ $c->others } ) {
      my $file_path = $f->name;
      my $file_size = $f->size;
      my $file_type = $f->type;
      my $file_md5  = $f->md5;
      my $host_id   = $f->host_id;
      
      next COLLECTION if $f->withdrawn eq 1;        ## not moving withdrawn files

      my $host = $ha->fetch_by_dbID($host_id);
      next COLLECTION if $host->remote eq 1;        ## not moving remote files

      my $new_path;
  
      my %new_path_hash = ( path_hash        => \%file_path_hash, 
                            metadata         => $meta_hash,
                            collection_name  => $collection_name,
                            file_path        => $file_path,
                            type             => $type,
                          );

      if ( $collection_name =~ /^ERR\d+/ ){         ## collections with ERR ids
        die "Run metadata file not defined"
           unless $run_metadata_file;
 
        $new_path_hash{ change_name } = 0;          ## keeping existing file name for run types
        $new_path = get_new_path( \%new_path_hash );
      }
      elsif ( $collection_name =~ /^ERX\d+/ ){      ## collections with ERX ids
        die "Exp metadata file not defined"
           unless $exp_metadata_file;

        $new_path_hash{ change_name } = 1;          ## changing file name for run types
        $new_path = get_new_path( \%new_path_hash );
      }
      throw("check collection name: $collection_name ,type: $file_type ,file: $file_path") if !$new_path;

      print join("\t", $file_path, $new_path),$/
          if $file_path ne $new_path;
    }
  }
}


sub get_list {
  my ( $file ) = @_;
  my @list;

  open my $fh, '<', $file;
  while( <$fh> ){
    chomp;
    push @list,$_;
  } 
  return \@list;
}

sub get_index_hash {
my ( $file, $key_string ) = @_;
  die "Key string not found to look in $file",$/
    unless $key_string;

  open my $fh, '<', $file;
  my @header;
  my %data;
  my $key_index = undef;

  while ( <$fh> ) {
    chomp;
    next if m/^#/;
    my @vals = split "\t", $_;

    if ( @header ) {
      $data { $vals[$key_index] }{ $header[$_] } = $vals[$_] for 0..$#header;
    }
    else {
      @header = map { uc($_) } @vals;
      my @key_index_array = grep{ $header[$_] eq $key_string } 0..$#header;

      die "$key_string not found in $file",$/
         if @key_index_array == 0;

      $key_index = $key_index_array[0];
    }
{'fraction_non_alphanumeric': 0.144156346749226, 'fraction_numerical': 0.0027089783281733747, 'mean_word_length': 1.8626419274439214, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2469078', 'n_tokens_mistral': 3713, 'n_tokens_neox': 3549, 'n_words': 789}