<commit_before>package JobDB::Job;

use strict;
use Data::Dumper;

use FIG_Config;
use DirHandle;
use File::Basename;
use IO::File;
use Fcntl ':flock';
use MGRAST::Metadata;

=pod

=head1 NAME

Job - MGRAST job access module

=head1 DESCRIPTION

TODO

=head1 METHODS

=over 4

=item * B<init> ()

Initialise a new instance of the Job object.

=cut

sub init {
  my $self = shift;

  # check if we are called properly
  unless (ref $self) {
    die "Not called as an object method.";
  }

  # parameters for the Job->init() call
  my $data = $_[0];
  unless (ref $data eq 'HASH') {
    die "Init without a parameters hash.";
  }

  my $job = $self->SUPER::init(@_);
  unless (ref $job) {
    print STDERR "Job init call failed.\n";
    return undef;
  }

  # # check if the user has the right to see this job
  unless ($job->public) {
    unless ( $job->_master->{_user} && $job->_master->{_user}->has_right(undef, 'view' , 'metagenome', $job->metagenome_id  ) ){
      return undef;
    }
  }
  return $job;
}

sub name {
  my ($self , $value) = @_ ;
  
  if ($value){
    $self->SUPER::name($value);
  }
  else{
    my $name = '';
    unless ($self->SUPER::name()){
      my $sample = $self->sample ;
      if ($sample and ref $sample){
	$name = join ";" , @{ $sample->data('sample_name') } ;
	$self->SUPER::name($name) if ($name) ;
      }
    }
  }

  return $self->SUPER::name() || 'unknown';
}

sub initialize {
  my ($self, $user, $data) = @_;

  my $master = $self->_master();
  unless (ref($master)) {
    print STDRER "reserve_job called without a dbmaster reference";
    return undef;
  }
  unless (ref($user)) {
    print STDRER "reserve_job called without a user";
    return undef;
  }

  my $dbh = $master->db_handle;
  my $max = $dbh->selectrow_arrayref("SELECT max(job_id + 0), max(metagenome_id + 0) FROM Job");
  my $job_id  = $max->[0] + 1;
  my $mg_id   = $max->[1] + 1;
  my $options = { owner => $user, job_id => $job_id, metagenome_id => $mg_id, server_version => 3 };
  my $params  = {};
 
  if (ref($data) eq "HASH") {
    $params = $data;
  }
  elsif ((! ref($data)) && (-s $data)) {
    my @lines = `cat $data`;
    chomp @lines;
    foreach my $line (@lines) {
      my ($k, $v) = split(/\t/, $line);
      $params->{$k} = $v;
    }
  }
  
  my $job_keys  = ['name','file','file_size','file_checksum','sequence_type'];
  my $stat_keys = ['bp_count','sequence_count','average_length','standard_deviation_length','length_min','length_max','average_gc_content','standard_deviation_gc_content','average_gc_ratio','standard_deviation_gc_ratio','ambig_char_count','ambig_sequence_count','average_ambig_chars','drisee_score'];

  foreach my $key (@$job_keys) {
    if (exists $params->{$key}) {
      if ($key =~ /^file_(size|checksum)/) {
	$options->{$key.'_raw'} = $params->{$key};
      } else {
	$options->{$key} = $params->{$key};
      }
    }
  }

  # Connect to User/Rights DB
  my $dbm = DBMaster->new(-database => $FIG_Config::webapplication_db,
			  -backend  => $FIG_Config::webapplication_backend,
			  -host     => $FIG_Config::webapplication_host,
			  -user     => $FIG_Config::webapplication_user,
			 );
  # check rights
  my $rights = ['view', 'edit', 'delete'];
  foreach my $right_name (@$rights) {
    my $objs = $dbm->Rights->get_objects({ scope     => $user->get_user_scope,
					   data_type => 'metagenome',
					   data_id   => $mg_id,
					   name      => $right_name,
					   granted   => 1 });
    unless (@$objs > 0) {
      my $right = $dbm->Rights->create({ scope     => $user->get_user_scope,
					 data_type => 'metagenome',
					 data_id   => $mg_id,
					 name      => $right_name,
					 granted   => 1 });
      unless (ref $right) {
	print STDRER "Unable to create Right $right_name - metagenome - $mg_id.";
	return undef;
      }
    }
  }

  ### now create job
  my $job = $master->Job->create($options);
  unless (ref $job) {
    print STDRER "Can't create job\n";
    return undef;
  }
  
  # store raw stats
  foreach my $key (@$stat_keys) {
    if (exists $params->{$key}) {
      $master->JobStatistics->create({ job => $job, tag => $key.'_raw', value => $params->{$key} });
    } elsif (exists $params->{$key.'_raw'}) {
      $master->JobStatistics->create({ job => $job, tag => $key.'_raw', value => $params->{$key.'_raw'} });
    }
  }

  # store attributes
  my $used_keys = {};
  map { $used_keys->{$_} = 1 } @$job_keys;
  map { $used_keys->{$_} = 1 } @$stat_keys;
  map { $used_keys->{$_.'_raw'} = 1 } @$stat_keys;
  
  foreach my $key (keys %$params) {
    next if (exists $used_keys->{$key});
    $key =~ s/\s+/_/g;
    $params->{$key} =~ s/\s+/_/g;
    $master->JobAttributes->create({ job => $job, tag => $key, value => $params->{$key} });
  }
  $job->set_filter_options();

  return $job;
}

sub reserve_job {
  my ($self, $user, $options, $stats) = @_;

  my $data = {};
  if (ref $options) {
    map { $data->{$_} = $options->{$_}; } keys %$options;
  }
  if (ref $stats) {
    map { $data->{$_} = $stats->{$_}; } keys %$stats;
  }
  return $self->initalize($user, $data);
}

sub has_checksum {
  my ($self, $checksum, $user) = @_;
  my $dbh = $self->_master->db_handle;
  my $md5 = $dbh->selectcol_arrayref("SELECT count(*) FROM Job WHERE file_checksum_raw='$checksum'".(($user && ref($user)) ? " AND owner=".$user->_id : ""));
  return ($md5 && @$md5 && ($md5->[0] > 0)) ? 1 : 0;  
}

sub finish_upload {
  my ($self, $file, $file_format) = @_ ;
  
  # create_and_submit_job -j <job_number> -f <sequence_file> [ -p <pipeline_name> -o <pipeline_options> --fastq --rna_only ]
  # set options 
  my $opts   = $self->set_job_options;
  my $format = ($file_format =~ /fastq/) ? "--fastq" : '' ;
  my $cmd    = $FIG_Config::create_job;
  my $params = " -j " . $self->job_id . " -f $file -o '$opts' $format";

  print STDERR "Calling $cmd $params\n";
  
  if ($cmd and -f $cmd) {
    my $output = `$cmd $params`;
    print STDERR $output;
    return $output;
  }
  else {
    print STDERR "Can't find $cmd\n";
    return 0;
  }
}

=pod 

=item * B<directory> ()

Returns the full path the job directory (without a trailing slash).

=cut

sub directory {
  my ($self) = @_;
  return $FIG_Config::mgrast_jobs.'/'.$self->job_id;
}

sub dir {
  my ($self) = @_;
  return $self->directory;
}

=pod 

=item * B<download_dir> ()

Returns the full path the download directory inside the job (without a trailing slash).

=cut

sub download_dir {
    my ($self, $stage) = @_ ;
    if ($stage) {
	return $self->directory.'/analysis/';
    }
    return $self->directory.'/raw/';
}

=pod 

=item * B<analysis_dir> ()

Returns the full path the analysis directory inside the job (without a trailing slash).

=cut

sub analysis_dir {
  my ($self) = @_;
  unless (-d $self->directory.'/analysis') {
    chdir($self->directory) or 
      die("Unable to change directory to ".$self->directory.": $!");
    mkdir "analysis", 0777 or 
      die("Unable to create directory analysis in ".$self->directory.": $!");
  }
  return $self->directory.'/analysis';
}

=pod

=item * B<download> ()

Returns the name of the project

=cut

sub download {
  my ($self , $stage_id , $file) = @_;

  if ($file) {
    if (open(FH, $self->download_dir($stage_id) . "/" . $file)) {
      print "Content-Type:application/x-download\n";  
      # print "Content-Length: " . length($content) . "\n";
      print "Content-Disposition:attachment;filename=".$self->metagenome_id. "." . $file ."\n\n";
      #print "<file name='".$self->metagenome_id. "." . $file ."'>";
      while (<FH>) {
	print $_ ;
      }
      return ( 1 , "" ) ;
    }
    else{
      return ( 0 , "Could not open download file " . $self->download_dir($stage_id) ."'$file'" );
    }
  }
  elsif (defined $stage_id){
    # Download uploaded files
    unless ($stage_id){ 
      
      opendir(DIR ,  $self->download_dir() ) ;
      while (my $file = readdir DIR ){
	next unless ($file =~/\.fna|\.fasta|\.sff|\.fastq|\.txt/) ;
	print STDERR "Downloading file $file";
	$self->download( '' , $file);
      }
      
    }
    
    return ( 1 , "" ) ;
  }
  else{
    # return list of download files
  }
  
  return 1;
}

=pod

=item * B<get_jobs_for_user> (I<user>, I<right>, I<viewable>)

Returns the Jobs objects the user I<user> has access to. Access to a job is defined
by the right to edit a genome of a certain metagenome_id. In the context of the RAST
server this method checks the 'edit - genome' rights of a user. Optionally, you can
change this by providing the parameter I<right> and setting it to eg. 'view'.
If present and true, the parameter I<viewable> restricts the query to jobs marked 
as viewable.

Please note you may not longer pass a scope to this function.

=cut

sub get_jobs_for_user {
  my ($self, $user, $right, $viewable) = @_;

  unless (ref $self) {
    die "Call method via the DBMaster.\n";
  }
 
  unless (ref $user and ( $user->isa("WebServerBackend::User"))) {
    print STDERR "No user given in method get_jobs_for_user.\n";
    die "No user given in method get_jobs_for_user.\n";
  }
  
  my $get_options = {};
  $get_options->{viewable} = 1 if ($viewable);
  my $right_to = $user->has_right_to(undef, $right || 'edit', 'metagenome');

  # check if first right_to is place holder
  if (scalar(@$right_to) and $right_to->[0] eq '*') {
    return $self->_master->Job->get_objects($get_options);
  } 
  
  $get_options->{owner} = $user;
  my $jobs = $self->_master->Job->get_objects($get_options);
  
  my %ids = map { $_ => 1 } @$right_to;
  foreach my $j (@$jobs){
    if ($j->metagenome_id){
		if (defined $ids{$j->metagenome_id}){
			delete $ids{$j->metagenome_id};
		}
    }
  }

  foreach (keys %ids){
	  my $tmp_j;
	  if($_){
		  $tmp_j = $self->_master->Job->init({ metagenome_id => $_ });
	  }
	  if($tmp_j){
		  if($viewable){
			  next unless $tmp_j->viewable;
		  } 
		  push @$jobs, $tmp_j;
	  } 
  }
  
  return $jobs;
}

sub get_jobs_for_user_fast {
    my ($self, $user_or_scope, $right, $viewable) = @_;
    
    unless (ref $self) {
	die "Call method via the DBMaster.\n";
    }
    
    unless (ref $user_or_scope and  ( $user_or_scope->isa("WebServerBackend::User") or $user_or_scope->isa("WebServerBackend::Scope"))) {
		print STDERR "No user or scope given in method get_jobs_for_user.\n";
		die "No user or scope given in method get_jobs_for_user.\n";
    }
    
    my $right_to = $user_or_scope->has_right_to(undef, $right || 'edit', 'metagenome');
    my $job_cond = "";

    if ($viewable) {
      $job_cond .= " AND viewable = 1";
    }
    unless (@$right_to and $right_to->[0] eq '*') {
      my @g = grep { $_ ne '*' } @$right_to;
      if (@g == 0) { return (); }
      $job_cond .= " AND metagenome_id IN ( " . join(", ", map { "'$_'" } @g) . ")";
    }

    my $dbh  = $self->_master()->db_handle();
    my $skip = $dbh->selectcol_arrayref(qq(SELECT DISTINCT job FROM JobAttributes WHERE tag='deleted' OR tag='no_sims_found'));
    if ($skip && @$skip) {
      $job_cond .= " AND j._id NOT IN (".join(",", @$skip).")";
    }
    my $res = $dbh->selectall_arrayref(qq(SELECT j.job_id, j.metagenome_id, j.name,
					  	j.file_size_raw, j.server_version,
					  	j.created_on, j.owner, j._owner_db, j.viewable, j.sequence_type,
					  	s.stage, s.status, s.timestamp, j._id
					  FROM Job j, PipelineStage s
					  WHERE s.job=j._id $job_cond
					  ORDER BY j.job_id DESC, s.timestamp ASC));
    my $ent = shift(@$res);
    my @out;
    while ($ent)
    {
		my($cur, $cur_genome, $cur_name, $cur_size, $cur_server_version, $cur_created, $cur_owner, $cur_owner_db, $jviewable, $jsequence_type, undef, undef, undef, $jid) = @$ent;
		my $stages = {};
		my $timed_stati = [];
		while ($ent and $ent->[0] eq $cur)
		{
			my($id, $genome, $name, $size, $vers, $created, $owner, $owner_db, $view, $type, $stage, $stat, $ts) = @$ent;
			
			$stages->{$stage} = $stat;
			push(@$timed_stati, [ $ts, $stage, $stat ]);
			$ent = shift(@$res);
		}
		push(@out, {
			    job_id => $cur,
			    metagenome_id => $cur_genome,
			    name => $cur_name,
			    project_name => '',
			    created_on => $cur_created,
			    status => $stages,
			    owner => $cur_owner,
			    owner_db => $cur_owner_db,
			    size => $cur_size,
			    server_version => $cur_server_version,
			    viewable => $jviewable,
			    _id => $jid,
			    sequence_type => $jsequence_type,
			    timed_stati => $timed_stati
			 });
    }
    return @out;
}

# new method section

=pod

=item * B<stats> ()

Returns a hash of all stats keys and values for a job. 
If a key is given , returns only hash of specified key, value pair.
Sets a value if key and value is given (return true or false if works)

=cut

sub stats {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if (defined($value) and $tag) {
    my $jstat = $self->_master->JobStatistics->get_objects( { job   => $self,
							      tag   => $tag,
							      value => $value
							    });
    if (ref $jstat and scalar @$jstat) {
      $jstat->[0]->value($value);
    }
    else{
      $jstat = $self->_master->JobStatistics->create( { job   => $self,
							tag   => $tag,
							value => $value
						      });
    }
    return 1;
  }
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT tag , value FROM JobStatistics where job=".$self->_id." and tag='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT tag , value FROM JobStatistics where job=".$self->_id);
  }
  
  $sth->execute;
  my $results = $sth->fetchall_arrayref();
  my $rhash   = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results ;
  
  return $rhash;
}

sub get_deleted_jobs {
  my ($self) = @_;

  my $del  = $self->_master->JobAttributes->get_objects({tag => 'deleted'});
  my @jobs = map { $_->job } @$del;

  return \@jobs;
}

sub get_bad_jobs {
  my ($self) = @_;

  my $bad  = $self->_master->JobAttributes->get_objects({tag => 'no_sims_found'});
  my @jobs = map { $_->job } @$bad;

  return \@jobs;
}

=pod

=item * B<stage> ()

Returns a hash of all stages and current status for a given job id.

=cut

sub get_stages_fast {
  my ($self, $jobid) = @_;

  unless (ref $self) {
    die "Call method via the DBMaster.\n";
  }

  my $dbh = $self->_master()->db_handle();
  my $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=$jobid");
  $sth->execute;

  my $rhash = {};
  my $rows  = $sth->fetchall_arrayref();
  if ($rows && (@$rows > 0)) {
    %$rhash = map { $_->[0], $_->[1] } @$rows;
  }
  return $rhash;
}

sub stage_info {
  my ($self, $tag, $value) = @_;
  
  $self->stage($tag, $value);
}

=pod

=item * B<stage> ()

Returns a hash of all stages and current status for a job.
If a stage is given, returns only hash of specified stage, current status.
Insert or update a status if stage and status is given.

=cut

sub stage {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if ($value and $tag) {
    my $jstat = $self->_master->PipelineStage->get_objects( { job    => $self,
							      stage  => $tag,
							      status => $value,
							    });
    # update existing stage-status
    if ( ref($jstat) and scalar(@$jstat) ) {
      my $time = $self->get_timestamp();
      $jstat->[0]->value($value);
      $jstat->[0]->timestamp($time);
    }
    # insert new stage-status
    else{ 
      $jstat = $self->_master->PipelineStage->create( { job    => $self,
							stage  => $tag,
							status => $value,
						      });
    }
    return { $tag => $value };
  }
  
  # get current status for input stage
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=" . $self->_id . " and stage='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=" . $self->_id);
  }
  $sth->execute;

  my $results = $sth->fetchall_arrayref();
  my $rhash = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results;
  
  return $rhash;
}

=pod

=item * B<data> ()

Returns a hash of all attribute keys and values for a job. 
If a key is given , returns only hash of specified key, value pair.
Sets a value if key and value is given (return true or false if works)

=cut

sub data {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if (defined($value) and $tag) {

    if (ref $value){
      print STDERR "ERROR: invalid value type for $tag  ($value) \n";
      print STDERR Dumper $value;
      return 0;
    }
    my $jstat = $self->_master->JobAttributes->get_objects( { job   => $self,
							      tag   => $tag,
							      value => $value
							    });
    if (ref $jstat and scalar @$jstat) {
      $jstat->[0]->value($value);
    }
    else {
      $jstat = $self->_master->JobAttributes->create( { job   => $self,
							tag   => $tag,
							value => $value
						      });
    }
    return 1;
  }
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT tag, value FROM JobAttributes where job=". $self->_id ." and tag='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT tag, value FROM JobAttributes where job=". $self->_id);
  }
  
  $sth->execute;
  my $results = $sth->fetchall_arrayref();
  my $rhash   = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results;
  
  return $rhash;
}

=pod

=item * B<set_filter_options> ()

job function that checks JobAttributes for a given set of tags,
if exist then creates new tag 'filter_options' with concatanated string,
returns filter_options string

=cut

sub set_filter_options {
  my ($self) = @_;

  my $flags = { filter_ln    => [ 'min_ln', 'max_ln' ],
		filter_ambig => [ 'max_ambig' ],
		dynamic_trim => [ 'min_qual', 'max_lqb' ]
	      };
  my %tags = map { $_, 1 } map { ($_, @{$flags->{$_}}) } keys %$flags;
  my @opts = ();
  my $data = $self->data();

  foreach my $t ( keys %tags ) {
    $tags{$t} = exists($data->{$t}) ? $data->{$t} : 0;
  }

  foreach my $f ( keys %$flags ) {
    if ( $tags{$f} ) {
      push @opts, $f;
      foreach my $s ( @{$flags->{$f}} ) { push @opts, "$s=" . $tags{$s}; }
    }
  }
  my $opts_string = join(":", @opts);
  $self->data('filter_options', $opts_string);
<commit_msg>fixed bug where missing pipeline options may break sequence filtering<commit_after>package JobDB::Job;

use strict;
use Data::Dumper;

use FIG_Config;
use DirHandle;
use File::Basename;
use IO::File;
use Fcntl ':flock';
use MGRAST::Metadata;

=pod

=head1 NAME

Job - MGRAST job access module

=head1 DESCRIPTION

TODO

=head1 METHODS

=over 4

=item * B<init> ()

Initialise a new instance of the Job object.

=cut

sub init {
  my $self = shift;

  # check if we are called properly
  unless (ref $self) {
    die "Not called as an object method.";
  }

  # parameters for the Job->init() call
  my $data = $_[0];
  unless (ref $data eq 'HASH') {
    die "Init without a parameters hash.";
  }

  my $job = $self->SUPER::init(@_);
  unless (ref $job) {
    print STDERR "Job init call failed.\n";
    return undef;
  }

  # # check if the user has the right to see this job
  unless ($job->public) {
    unless ( $job->_master->{_user} && $job->_master->{_user}->has_right(undef, 'view' , 'metagenome', $job->metagenome_id  ) ){
      return undef;
    }
  }
  return $job;
}

sub name {
  my ($self , $value) = @_ ;
  
  if ($value){
    $self->SUPER::name($value);
  }
  else{
    my $name = '';
    unless ($self->SUPER::name()){
      my $sample = $self->sample ;
      if ($sample and ref $sample){
	$name = join ";" , @{ $sample->data('sample_name') } ;
	$self->SUPER::name($name) if ($name) ;
      }
    }
  }

  return $self->SUPER::name() || 'unknown';
}

sub initialize {
  my ($self, $user, $data) = @_;

  my $master = $self->_master();
  unless (ref($master)) {
    print STDRER "reserve_job called without a dbmaster reference";
    return undef;
  }
  unless (ref($user)) {
    print STDRER "reserve_job called without a user";
    return undef;
  }

  my $dbh = $master->db_handle;
  my $max = $dbh->selectrow_arrayref("SELECT max(job_id + 0), max(metagenome_id + 0) FROM Job");
  my $job_id  = $max->[0] + 1;
  my $mg_id   = $max->[1] + 1;
  my $options = { owner => $user, job_id => $job_id, metagenome_id => $mg_id, server_version => 3 };
  my $params  = {};
 
  if (ref($data) eq "HASH") {
    $params = $data;
  }
  elsif ((! ref($data)) && (-s $data)) {
    my @lines = `cat $data`;
    chomp @lines;
    foreach my $line (@lines) {
      my ($k, $v) = split(/\t/, $line);
      $params->{$k} = $v;
    }
  }
  
  my $job_keys  = ['name','file','file_size','file_checksum','sequence_type'];
  my $stat_keys = ['bp_count','sequence_count','average_length','standard_deviation_length','length_min','length_max','average_gc_content','standard_deviation_gc_content','average_gc_ratio','standard_deviation_gc_ratio','ambig_char_count','ambig_sequence_count','average_ambig_chars','drisee_score'];

  foreach my $key (@$job_keys) {
    if (exists $params->{$key}) {
      if ($key =~ /^file_(size|checksum)/) {
	$options->{$key.'_raw'} = $params->{$key};
      } else {
	$options->{$key} = $params->{$key};
      }
    }
  }

  # Connect to User/Rights DB
  my $dbm = DBMaster->new(-database => $FIG_Config::webapplication_db,
			  -backend  => $FIG_Config::webapplication_backend,
			  -host     => $FIG_Config::webapplication_host,
			  -user     => $FIG_Config::webapplication_user,
			 );
  # check rights
  my $rights = ['view', 'edit', 'delete'];
  foreach my $right_name (@$rights) {
    my $objs = $dbm->Rights->get_objects({ scope     => $user->get_user_scope,
					   data_type => 'metagenome',
					   data_id   => $mg_id,
					   name      => $right_name,
					   granted   => 1 });
    unless (@$objs > 0) {
      my $right = $dbm->Rights->create({ scope     => $user->get_user_scope,
					 data_type => 'metagenome',
					 data_id   => $mg_id,
					 name      => $right_name,
					 granted   => 1 });
      unless (ref $right) {
	print STDRER "Unable to create Right $right_name - metagenome - $mg_id.";
	return undef;
      }
    }
  }

  ### now create job
  my $job = $master->Job->create($options);
  unless (ref $job) {
    print STDRER "Can't create job\n";
    return undef;
  }
  
  # store raw stats
  foreach my $key (@$stat_keys) {
    if (exists $params->{$key}) {
      $master->JobStatistics->create({ job => $job, tag => $key.'_raw', value => $params->{$key} });
    } elsif (exists $params->{$key.'_raw'}) {
      $master->JobStatistics->create({ job => $job, tag => $key.'_raw', value => $params->{$key.'_raw'} });
    }
  }

  # store attributes
  my $used_keys = {};
  map { $used_keys->{$_} = 1 } @$job_keys;
  map { $used_keys->{$_} = 1 } @$stat_keys;
  map { $used_keys->{$_.'_raw'} = 1 } @$stat_keys;
  
  foreach my $key (keys %$params) {
    next if (exists $used_keys->{$key});
    $key =~ s/\s+/_/g;
    $params->{$key} =~ s/\s+/_/g;
    $master->JobAttributes->create({ job => $job, tag => $key, value => $params->{$key} });
  }
  $job->set_filter_options();

  return $job;
}

sub reserve_job {
  my ($self, $user, $options, $stats) = @_;

  my $data = {};
  if (ref $options) {
    map { $data->{$_} = $options->{$_}; } keys %$options;
  }
  if (ref $stats) {
    map { $data->{$_} = $stats->{$_}; } keys %$stats;
  }
  return $self->initalize($user, $data);
}

sub has_checksum {
  my ($self, $checksum, $user) = @_;
  my $dbh = $self->_master->db_handle;
  my $md5 = $dbh->selectcol_arrayref("SELECT count(*) FROM Job WHERE file_checksum_raw='$checksum'".(($user && ref($user)) ? " AND owner=".$user->_id : ""));
  return ($md5 && @$md5 && ($md5->[0] > 0)) ? 1 : 0;  
}

sub finish_upload {
  my ($self, $file, $file_format) = @_ ;
  
  # create_and_submit_job -j <job_number> -f <sequence_file> [ -p <pipeline_name> -o <pipeline_options> --fastq --rna_only ]
  # set options 
  my $opts   = $self->set_job_options;
  my $format = ($file_format =~ /fastq/) ? "--fastq" : '' ;
  my $cmd    = $FIG_Config::create_job;
  my $params = " -j " . $self->job_id . " -f $file -o '$opts' $format";

  print STDERR "Calling $cmd $params\n";
  
  if ($cmd and -f $cmd) {
    my $output = `$cmd $params`;
    print STDERR $output;
    return $output;
  }
  else {
    print STDERR "Can't find $cmd\n";
    return 0;
  }
}

=pod 

=item * B<directory> ()

Returns the full path the job directory (without a trailing slash).

=cut

sub directory {
  my ($self) = @_;
  return $FIG_Config::mgrast_jobs.'/'.$self->job_id;
}

sub dir {
  my ($self) = @_;
  return $self->directory;
}

=pod 

=item * B<download_dir> ()

Returns the full path the download directory inside the job (without a trailing slash).

=cut

sub download_dir {
    my ($self, $stage) = @_ ;
    if ($stage) {
	return $self->directory.'/analysis/';
    }
    return $self->directory.'/raw/';
}

=pod 

=item * B<analysis_dir> ()

Returns the full path the analysis directory inside the job (without a trailing slash).

=cut

sub analysis_dir {
  my ($self) = @_;
  unless (-d $self->directory.'/analysis') {
    chdir($self->directory) or 
      die("Unable to change directory to ".$self->directory.": $!");
    mkdir "analysis", 0777 or 
      die("Unable to create directory analysis in ".$self->directory.": $!");
  }
  return $self->directory.'/analysis';
}

=pod

=item * B<download> ()

Returns the name of the project

=cut

sub download {
  my ($self , $stage_id , $file) = @_;

  if ($file) {
    if (open(FH, $self->download_dir($stage_id) . "/" . $file)) {
      print "Content-Type:application/x-download\n";  
      # print "Content-Length: " . length($content) . "\n";
      print "Content-Disposition:attachment;filename=".$self->metagenome_id. "." . $file ."\n\n";
      #print "<file name='".$self->metagenome_id. "." . $file ."'>";
      while (<FH>) {
	print $_ ;
      }
      return ( 1 , "" ) ;
    }
    else{
      return ( 0 , "Could not open download file " . $self->download_dir($stage_id) ."'$file'" );
    }
  }
  elsif (defined $stage_id){
    # Download uploaded files
    unless ($stage_id){ 
      
      opendir(DIR ,  $self->download_dir() ) ;
      while (my $file = readdir DIR ){
	next unless ($file =~/\.fna|\.fasta|\.sff|\.fastq|\.txt/) ;
	print STDERR "Downloading file $file";
	$self->download( '' , $file);
      }
      
    }
    
    return ( 1 , "" ) ;
  }
  else{
    # return list of download files
  }
  
  return 1;
}

=pod

=item * B<get_jobs_for_user> (I<user>, I<right>, I<viewable>)

Returns the Jobs objects the user I<user> has access to. Access to a job is defined
by the right to edit a genome of a certain metagenome_id. In the context of the RAST
server this method checks the 'edit - genome' rights of a user. Optionally, you can
change this by providing the parameter I<right> and setting it to eg. 'view'.
If present and true, the parameter I<viewable> restricts the query to jobs marked 
as viewable.

Please note you may not longer pass a scope to this function.

=cut

sub get_jobs_for_user {
  my ($self, $user, $right, $viewable) = @_;

  unless (ref $self) {
    die "Call method via the DBMaster.\n";
  }
 
  unless (ref $user and ( $user->isa("WebServerBackend::User"))) {
    print STDERR "No user given in method get_jobs_for_user.\n";
    die "No user given in method get_jobs_for_user.\n";
  }
  
  my $get_options = {};
  $get_options->{viewable} = 1 if ($viewable);
  my $right_to = $user->has_right_to(undef, $right || 'edit', 'metagenome');

  # check if first right_to is place holder
  if (scalar(@$right_to) and $right_to->[0] eq '*') {
    return $self->_master->Job->get_objects($get_options);
  } 
  
  $get_options->{owner} = $user;
  my $jobs = $self->_master->Job->get_objects($get_options);
  
  my %ids = map { $_ => 1 } @$right_to;
  foreach my $j (@$jobs){
    if ($j->metagenome_id){
		if (defined $ids{$j->metagenome_id}){
			delete $ids{$j->metagenome_id};
		}
    }
  }

  foreach (keys %ids){
	  my $tmp_j;
	  if($_){
		  $tmp_j = $self->_master->Job->init({ metagenome_id => $_ });
	  }
	  if($tmp_j){
		  if($viewable){
			  next unless $tmp_j->viewable;
		  } 
		  push @$jobs, $tmp_j;
	  } 
  }
  
  return $jobs;
}

sub get_jobs_for_user_fast {
    my ($self, $user_or_scope, $right, $viewable) = @_;
    
    unless (ref $self) {
	die "Call method via the DBMaster.\n";
    }
    
    unless (ref $user_or_scope and  ( $user_or_scope->isa("WebServerBackend::User") or $user_or_scope->isa("WebServerBackend::Scope"))) {
		print STDERR "No user or scope given in method get_jobs_for_user.\n";
		die "No user or scope given in method get_jobs_for_user.\n";
    }
    
    my $right_to = $user_or_scope->has_right_to(undef, $right || 'edit', 'metagenome');
    my $job_cond = "";

    if ($viewable) {
      $job_cond .= " AND viewable = 1";
    }
    unless (@$right_to and $right_to->[0] eq '*') {
      my @g = grep { $_ ne '*' } @$right_to;
      if (@g == 0) { return (); }
      $job_cond .= " AND metagenome_id IN ( " . join(", ", map { "'$_'" } @g) . ")";
    }

    my $dbh  = $self->_master()->db_handle();
    my $skip = $dbh->selectcol_arrayref(qq(SELECT DISTINCT job FROM JobAttributes WHERE tag='deleted' OR tag='no_sims_found'));
    if ($skip && @$skip) {
      $job_cond .= " AND j._id NOT IN (".join(",", @$skip).")";
    }
    my $res = $dbh->selectall_arrayref(qq(SELECT j.job_id, j.metagenome_id, j.name,
					  	j.file_size_raw, j.server_version,
					  	j.created_on, j.owner, j._owner_db, j.viewable, j.sequence_type,
					  	s.stage, s.status, s.timestamp, j._id
					  FROM Job j, PipelineStage s
					  WHERE s.job=j._id $job_cond
					  ORDER BY j.job_id DESC, s.timestamp ASC));
    my $ent = shift(@$res);
    my @out;
    while ($ent)
    {
		my($cur, $cur_genome, $cur_name, $cur_size, $cur_server_version, $cur_created, $cur_owner, $cur_owner_db, $jviewable, $jsequence_type, undef, undef, undef, $jid) = @$ent;
		my $stages = {};
		my $timed_stati = [];
		while ($ent and $ent->[0] eq $cur)
		{
			my($id, $genome, $name, $size, $vers, $created, $owner, $owner_db, $view, $type, $stage, $stat, $ts) = @$ent;
			
			$stages->{$stage} = $stat;
			push(@$timed_stati, [ $ts, $stage, $stat ]);
			$ent = shift(@$res);
		}
		push(@out, {
			    job_id => $cur,
			    metagenome_id => $cur_genome,
			    name => $cur_name,
			    project_name => '',
			    created_on => $cur_created,
			    status => $stages,
			    owner => $cur_owner,
			    owner_db => $cur_owner_db,
			    size => $cur_size,
			    server_version => $cur_server_version,
			    viewable => $jviewable,
			    _id => $jid,
			    sequence_type => $jsequence_type,
			    timed_stati => $timed_stati
			 });
    }
    return @out;
}

# new method section

=pod

=item * B<stats> ()

Returns a hash of all stats keys and values for a job. 
If a key is given , returns only hash of specified key, value pair.
Sets a value if key and value is given (return true or false if works)

=cut

sub stats {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if (defined($value) and $tag) {
    my $jstat = $self->_master->JobStatistics->get_objects( { job   => $self,
							      tag   => $tag,
							      value => $value
							    });
    if (ref $jstat and scalar @$jstat) {
      $jstat->[0]->value($value);
    }
    else{
      $jstat = $self->_master->JobStatistics->create( { job   => $self,
							tag   => $tag,
							value => $value
						      });
    }
    return 1;
  }
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT tag , value FROM JobStatistics where job=".$self->_id." and tag='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT tag , value FROM JobStatistics where job=".$self->_id);
  }
  
  $sth->execute;
  my $results = $sth->fetchall_arrayref();
  my $rhash   = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results ;
  
  return $rhash;
}

sub get_deleted_jobs {
  my ($self) = @_;

  my $del  = $self->_master->JobAttributes->get_objects({tag => 'deleted'});
  my @jobs = map { $_->job } @$del;

  return \@jobs;
}

sub get_bad_jobs {
  my ($self) = @_;

  my $bad  = $self->_master->JobAttributes->get_objects({tag => 'no_sims_found'});
  my @jobs = map { $_->job } @$bad;

  return \@jobs;
}

=pod

=item * B<stage> ()

Returns a hash of all stages and current status for a given job id.

=cut

sub get_stages_fast {
  my ($self, $jobid) = @_;

  unless (ref $self) {
    die "Call method via the DBMaster.\n";
  }

  my $dbh = $self->_master()->db_handle();
  my $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=$jobid");
  $sth->execute;

  my $rhash = {};
  my $rows  = $sth->fetchall_arrayref();
  if ($rows && (@$rows > 0)) {
    %$rhash = map { $_->[0], $_->[1] } @$rows;
  }
  return $rhash;
}

sub stage_info {
  my ($self, $tag, $value) = @_;
  
  $self->stage($tag, $value);
}

=pod

=item * B<stage> ()

Returns a hash of all stages and current status for a job.
If a stage is given, returns only hash of specified stage, current status.
Insert or update a status if stage and status is given.

=cut

sub stage {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if ($value and $tag) {
    my $jstat = $self->_master->PipelineStage->get_objects( { job    => $self,
							      stage  => $tag,
							      status => $value,
							    });
    # update existing stage-status
    if ( ref($jstat) and scalar(@$jstat) ) {
      my $time = $self->get_timestamp();
      $jstat->[0]->value($value);
      $jstat->[0]->timestamp($time);
    }
    # insert new stage-status
    else{ 
      $jstat = $self->_master->PipelineStage->create( { job    => $self,
							stage  => $tag,
							status => $value,
						      });
    }
    return { $tag => $value };
  }
  
  # get current status for input stage
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=" . $self->_id . " and stage='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT stage, status FROM PipelineStage where job=" . $self->_id);
  }
  $sth->execute;

  my $results = $sth->fetchall_arrayref();
  my $rhash = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results;
  
  return $rhash;
}

=pod

=item * B<data> ()

Returns a hash of all attribute keys and values for a job. 
If a key is given , returns only hash of specified key, value pair.
Sets a value if key and value is given (return true or false if works)

=cut

sub data {
  my ($self, $tag, $value) = @_;

  my $dbh = $self->_master->db_handle;
  my $sth;
  
  if (defined($value) and $tag) {

    if (ref $value){
      print STDERR "ERROR: invalid value type for $tag  ($value) \n";
      print STDERR Dumper $value;
      return 0;
    }
    my $jstat = $self->_master->JobAttributes->get_objects( { job   => $self,
							      tag   => $tag,
							      value => $value
							    });
    if (ref $jstat and scalar @$jstat) {
      $jstat->[0]->value($value);
    }
    else {
      $jstat = $self->_master->JobAttributes->create( { job   => $self,
							tag   => $tag,
							value => $value
						      });
    }
    return 1;
  }
  elsif ($tag) {
    $sth = $dbh->prepare("SELECT tag, value FROM JobAttributes where job=". $self->_id ." and tag='$tag'");
  }
  else {
    $sth = $dbh->prepare("SELECT tag, value FROM JobAttributes where job=". $self->_id);
  }
  
  $sth->execute;
  my $results = $sth->fetchall_arrayref();
  my $rhash   = {};
  map { $rhash->{ $_->[0] } = $_->[1] } @$results;
  
  return $rhash;
}

=pod

=item * B<set_filter_options> ()

job function that checks JobAttributes for a given set of tags,
if exist then creates new tag 'filter_options' with concatanated string,
returns filter_options string

=cut

sub set_filter_options {
  my ($self) = @_;

  my $flags = { filter_ln    => [ 'min_ln', 'max_ln' ],
		filter_ambig => [ 'max_ambig' ],
		dynamic_trim => [ 'min_qual', 'max_lqb' ]
	      };
  my %tags = map { $_, 1 } map { ($_, @{$flags->{$_}}) } keys %$flags;
  my @opts = ();
  my $data = $self->data();
  my $skip = 1;

  foreach my $t ( keys %tags ) {
    $tags{$t} = exists($data->{$t}) ? $data->{$t} : 0;
  }

  foreach my $f ( keys %$flags ) {
    if ( $tags{$f} ) {
      $skip = 0;
      push @opts, $f;
      foreach my $s ( @{$flags->{$f}} ) { push @opts, "$s=" . $tags{$s}; }
    }
  }
  my $opts_string = $skip ? 'skip' : join(":", @opts);
  $self->data('filter_options', $opts_string);

{'fraction_non_alphanumeric': 0.18318061382203576, 'fraction_numerical': 0.00336718922499448, 'mean_word_length': 2.8386481618815553, 'pattern_counts': {'":': 6, '<': 49, '<?xml version=': 0, '>': 567, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1272295', 'n_tokens_mistral': 14137, 'n_tokens_neox': 13281, 'n_words': 3456}