<commit_before>#!/usr/bin/env perl
use strict;
use warnings;
use Cwd;
use Path::Class qw(file dir);
use File::Copy;

BEGIN {
    use Test::Most tests => 106;
    
    use_ok('VRPipe::Persistent');
    use_ok('VRPipe::Persistent::Schema');
    
    use TestPersistentReal;
}

# some quick basic tests for all the core domain classes
my @schedulers;
ok $schedulers[0] = VRPipe::Scheduler->get(type => 'LSF'), 'created a Scheduler using get()';
is_deeply [$schedulers[0]->id, $schedulers[0]->type], [1, 'LSF'], 'scheduler1 has the expected fields';
ok $schedulers[1] = VRPipe::Scheduler->get(type => 'local'), 'created another Scheduler using get()';
is_deeply [$schedulers[1]->id, $schedulers[1]->type], [2, 'local'], 'scheduler2 has the expected fields';
ok my $default_type = VRPipe::Scheduler->default_type, 'could get a default type';
ok my $default_output_root = VRPipe::Scheduler->default_output_root, 'could get a default output root';
ok $schedulers[2] = VRPipe::Scheduler->get(), 'created another Scheduler using get() with no args';
is_deeply [$schedulers[2]->id, $schedulers[2]->type, $schedulers[2]->output_root], [1, $default_type, $default_output_root], 'scheduler3 has default fields';
$schedulers[2]->start_scheduler;

my $output_dir = dir($schedulers[2]->output_root, 'persistent_test_output');
$schedulers[2]->remove_tree($output_dir);
$schedulers[2]->make_path($output_dir);

my @files;
my $input1_path = file($output_dir, 'input1.txt');
open(my $fh, '>', $input1_path) or die "Could not write to $input1_path\n";
print $fh "input1_line1\ninput2_line2\n";
close($fh);
ok $files[0] = VRPipe::File->get(path => $input1_path, type => 'txt', metadata => {foo => 'bar'}), 'created a File using get()';
my $output1_path = file($output_dir, 'output1.txt');
ok $files[1] = VRPipe::File->get(path => $output1_path, type => 'txt'), 'created another File using get()';

my @ids;
ok $ids[0] = VRPipe::StepIODefinition->get(type => 'bam', description => 'step_1 bam input'), 'created a InputDefinition using get()';

my @steps;
ok $steps[0] = VRPipe::Step->get(name => 'step_1',
                                 inputs_definition => { static_input => $files[0],
                                                        dynamic_input => $ids[0] },
                                 body_sub => sub {
                                                    my $self = shift;
                                                    my $ofile = $self->output_file(output_key => 'step1_output', basename => 'output1.txt', type => 'txt');
                                                    my $fh = $ofile->openw();
                                                    print $fh "step1output\n";
                                                    $ofile->close();
                                                 },
                                 outputs_definition => { step1_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step1_output file') },
                                 post_process_sub => sub { return 1 },
                                 description => 'the first step'), 'created a Step using get()';
is_deeply [$steps[0]->id, $steps[0]->description], [1, 'the first step'], 'step1 has the expected fields';
undef $steps[0];
ok $steps[0] = VRPipe::Step->get(name => 'step_1'), 'got a Step using get(name => )';
is_deeply [$steps[0]->id,
           $steps[0]->description,
           $steps[0]->inputs_definition->{static_input}->path], [1, 'the first step', $input1_path], 'step1 still has the expected fields';

ok my $first_step = VRPipe::Step->get(id => 1), 'step1 could be gotten by id';
is $first_step->description, 'the first step', 'it has the correct description';
is $first_step->description('the 1st step'), 'the first step', 'description does not seem like it was changed prior to an update';
is $steps[0]->description, 'the first step', 'indeed, other instances are not affected either';
$first_step->update;
is $steps[0]->description, 'the 1st step', 'all instances are affected after an update';

my @jobs;
my $epoch_time = time();
ok $jobs[0] = VRPipe::Job->get(cmd => 'echo "job1";'), 'created a Job using get() with no dir';
is_deeply [$jobs[0]->id, $jobs[0]->cmd, $jobs[0]->dir, $jobs[0]->running], [1, 'echo "job1";', cwd(), 0], 'job1 has the expected fields';
undef $jobs[0];
$jobs[0] = VRPipe::Job->get(cmd => 'echo "job1";');
cmp_ok $jobs[0]->creation_time->epoch, '>=', $epoch_time, 'creation time defaulted to just now';
is $jobs[0]->start_time, undef, 'start_time defaults to undef';
$jobs[1] = VRPipe::Job->get(cmd => 'echo "job2";');
$jobs[1]->pid(33);
$jobs[1]->user('foo');
$jobs[1]->host('local');
$jobs[1]->update;
undef $jobs[1];
$jobs[1] = VRPipe::Job->get(id => 2);
is_deeply [$jobs[1]->id, $jobs[1]->pid, $jobs[1]->user, $jobs[1]->host], [2, 33, 'foo', 'local'], 'you can set multiple fields followed by a single update';
$jobs[1]->pid(undef);
$jobs[1]->update;
undef $jobs[1];
$jobs[1] = VRPipe::Job->get(id => 2);
is $jobs[1]->pid, undef, 'you can set a field to undef';

my @reqs;
ok $reqs[0] = VRPipe::Requirements->get(memory => 2000, time => 6), 'created a Requirments using get() with only memory and time';
undef $reqs[0];
$reqs[0] = VRPipe::Requirements->get(memory => 2000, time => 6);
is_deeply [$reqs[0]->id, $reqs[0]->memory, $reqs[0]->time, $reqs[0]->cpus, $reqs[0]->tmp_space, $reqs[0]->local_space, $reqs[0]->custom], [1, 2000, 6, 1, 0, 0, {}], 'reqs1 has the expected fields';
ok $reqs[1] = VRPipe::Requirements->get(memory => 2000, time => 6, cpus => 2, tmp_space => 500, local_space => 0, custom => {loo => 'car'}), 'created a Requirments using fully specified get()';
undef $reqs[1];
$reqs[1] = VRPipe::Requirements->get(memory => 2000, time => 6, cpus => 2, tmp_space => 500, local_space => 0, custom => {loo => 'car'});
is_deeply [$reqs[1]->id, $reqs[1]->memory, $reqs[1]->time, $reqs[1]->cpus, $reqs[1]->tmp_space, $reqs[1]->local_space, $reqs[1]->custom->{loo}], [2, 2000, 6, 2, 500, 0, 'car'], 'reqs2 has the expected fields and is a seperate new entry in the db';

my @ds;
ok $ds[0] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.fivelist'), 'created a DataSource using get()';
undef $ds[0];
$ds[0] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.fivelist');
is_fields [qw/id type method source/], $ds[0], [1, 'list', 'all', 't/data/datasource.fivelist'], 'ds1 has the expected fields';
$ds[1] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.onelist');
is_fields [qw/id type method source/], $ds[1], [2, 'list', 'all', 't/data/datasource.onelist'], 'ds2 has the expected fields';

my @de;
foreach my $de_num (1..5) {
    # create
    $ds[0]->next_element;
}
foreach my $de_num (1..5) {
    # get
    push(@de, VRPipe::DataElement->get(id => $de_num));
}
is_deeply [$de[2]->id, $de[2]->datasource->id, $de[2]->result->{line}], [3, 1, 'fed_result_3'], 'de3 has the expected fields';

my @pipelines;
ok $pipelines[0] = VRPipe::Pipeline->get(name => 'p1', description => 'first test pipeline'), 'created a Pipeline using get()';
undef $pipelines[0];
$pipelines[0] = VRPipe::Pipeline->get(name => 'p1', description => 'first test pipeline');
is_fields [qw/id name description/], $pipelines[0], [1, 'p1', 'first test pipeline'], 'pipeline1 has the expected fields';

# create some more steps we can chain together into a proper pipeline
VRPipe::Step->get(name => "step_2",
                  inputs_definition => { step2_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_2 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step2_output', basename => 'step2_output.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step2output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step2_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step2_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_3",
                  inputs_definition => { step3_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_3 input') },
                  body_sub => sub { 
                                    my $self = shift;
                                    my $ofile = $self->output_file(output_key => 'step3_output', basename => 'step3_output.txt', type => 'txt')->path;
                                    $self->dispatch(["sleep 5; echo step3output > $ofile", $self->new_requirements(memory => 50, time => 1)]);
                                    $self->dispatch(["sleep 4;", $self->new_requirements(memory => 50, time => 1)]);
                                    $self->dispatch(["sleep 3;", $self->new_requirements(memory => 50, time => 1)]);
                                   },
                  outputs_definition => { step3_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step3_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_4",
                  inputs_definition => { step4_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_4 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step4_output', basename => 'step4_basename.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step4output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step4_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step4_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_5",
                  inputs_definition => { step5_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_5 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step5_output', basename => 'step5_output.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step5output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step5_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step5_output file') },
                  post_process_sub => sub { return 1 });
foreach my $step_num (2..5) {
    # get
    push(@steps, VRPipe::Step->get(id => $step_num));
}
is_deeply [$steps[2]->id, $steps[2]->inputs_definition->{step3_input}->type, $steps[2]->outputs_definition->{step3_output}->description], [3, 'txt', 'step3_output file'], 'step3 has the expected fields';

# create a 5 step pipeline by creating stepmembers
my @stepms;
my $step_num = 0;
foreach my $step (@steps) {
    # create
    VRPipe::StepMember->get(step => $step, pipeline => $pipelines[0], step_number => ++$step_num);
}
for (1..5) {
    # get
    push(@stepms, VRPipe::StepMember->get(id => $_));
}
is_deeply [$stepms[2]->id, $stepms[2]->step->id, $stepms[2]->pipeline->id, $stepms[2]->step_number], [3, 3, 1, 3], 'stepmember3 has the expected fields';

my @setups;
my $step1_bam_input = file($output_dir, 'input.bam');
copy(file(qw(t data file.bam)), $step1_bam_input) || die "Could not copy to $step1_bam_input\n";
ok $setups[0] = VRPipe::PipelineSetup->get(name => 'ps1', datasource => $ds[0], output_root => $output_dir, pipeline => $pipelines[0], options => {dynamic_input => "$step1_bam_input", baz => 'loman'}), 'created a PipelineSetup using get()';
undef $setups[0];
$setups[0] = VRPipe::PipelineSetup->get(id => 1);
is_deeply [$setups[0]->id, $setups[0]->datasource->id, $setups[0]->pipeline->id, $setups[0]->options->{baz}, $setups[0]->options->{dynamic_input}], [1, 1, 1, 'loman', $step1_bam_input], 'pipelinesetup1 has the expected fields when retrievied with just id';
undef $setups[0];
$setups[0] = VRPipe::PipelineSetup->get(name => 'ps1', datasource => $ds[0], output_root => $output_dir, pipeline => $pipelines[0], options => {dynamic_input => "$step1_bam_input", baz => 'loman'});
is_deeply [$setups[0]->id, $setups[0]->datasource->id, $setups[0]->pipeline->id, $setups[0]->options->{baz}], [1, 1, 1, 'loman'], 'pipelinesetup1 has the expected fields when retrieved with a full spec';


ok my $first_setup = VRPipe::PipelineSetup->get(id => 1), 'pipelinesetup1 could be gotten by id';
is $first_setup->datasource->id, 1, 'it has the correct datasource';
$first_setup->datasource($ds[1]); # *** though we shouldn't be able to change an is_key value
$first_setup->update;
is $first_setup->datasource->id, 2, 'datasource could be changed';
is $setups[0]->datasource->id, 2, 'and since we did an update, other instances are affected as well';
$first_setup->datasource($ds[0]);
$first_setup->update;

my @stepstates;
ok $stepstates[0] = VRPipe::StepState->get(stepmember => $stepms[0], dataelement => $de[0], pipelinesetup => $setups[0]), 'created a StepState using get()';
undef $stepstates[0];
$stepstates[0] = VRPipe::StepState->get(id => 1);
is_deeply [$stepstates[0]->id, $stepstates[0]->stepmember->id, $stepstates[0]->dataelement->id, $stepstates[0]->pipelinesetup->id, $stepstates[0]->complete], [1, 1, 1, 1, 0], 'stepstate1 has the expected fields';
$stepstates[1] = VRPipe::StepState->get(stepmember => $stepms[0], dataelement => $de[1], pipelinesetup => $setups[0]);

my @subs;
throws_ok { VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0]) } qr/Attribute \(requirements\) is required/, 'requirements is required when created a Submission, even though it is not a key';
ok $subs[0] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0], requirements => $reqs[0]), 'Submission could be made without required scheduler, which can default';
undef $subs[0];
$subs[0] = VRPipe::Submission->get(id => 1);
is_deeply [$subs[0]->id, $subs[0]->job->cmd, $subs[0]->stepstate->id, $subs[0]->requirements->memory, $subs[0]->retries, $subs[0]->scheduled, $subs[0]->done], [1, 'echo "job1";', 1, 2000, 0, undef, 0], 'submission1 has the expected fields';
ok my $defaulted_scheduler = $subs[0]->scheduler, 'submission1 has a scheduler generated by default';
is $defaulted_scheduler->id, 3, 'The scheduler was the default one';
is $subs[0]->memory, 2000, 'requirement methods pass through to the requriements object';
ok $subs[0]->extra_memory(500), 'could set some extra memory';
is $subs[0]->memory, 2500, 'now our requirements object has more memory';
is $subs[0]->requirements->id, 3, 'and it is actually a new row in the db';
$subs[0]->update;
is $subs[0]->id, 1, 'yet the submission id did not change';
$subs[0]->extra_memory();
is $subs[0]->memory, 3500, 'extra_memory defaults to adding 1000MB';
$subs[1] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0], requirements => $reqs[1]);
is $subs[1]->id, 1, 'a Submission created with only the reqs differing results in the same submission';
undef $subs[1];
$subs[1] = VRPipe::Submission->get(id => 1);
is $subs[1]->requirements->id, 2, 'but getting it with a certain requirements changed the requirements for this submission in the db';
$subs[1] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[1], requirements => $reqs[1]);

throws_ok { VRPipe::PersistentArray->get() } qr/needs id or members/, 'get() for PersistentArray fails with no args';
throws_ok { VRPipe::PersistentArray->get(id => 1, members => \@subs) } qr/both id and members cannot be supplied/, 'get() for PersistentArray fails with both id and members supplied';
ok my $subs_array = VRPipe::PersistentArray->get(members => \@subs), 'created a PersistentArray using get(members => [...])';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [1, 1, 2], 'the created PArray has the correct contents';
undef $subs_array;
ok $subs_array = VRPipe::PersistentArray->get(id => 1), 'got a PersistentArray using get(id => 1)';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [1, 1, 2], 'the gotten PArray has the correct contents';
ok $subs_array = VRPipe::PersistentArray->get(members => \@subs), 'created a PersistentArray using the same set of members)';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [2, 1, 2], 'the created PArray has a new id, but the same contents otherwise';
is $subs_array->member(2)->id, 2, 'member() works given an index';
undef $subs_array;<commit_msg>corrected a test following scheduler change<commit_after>#!/usr/bin/env perl
use strict;
use warnings;
use Cwd;
use Path::Class qw(file dir);
use File::Copy;

BEGIN {
    use Test::Most tests => 106;
    
    use_ok('VRPipe::Persistent');
    use_ok('VRPipe::Persistent::Schema');
    
    use TestPersistentReal;
}

# some quick basic tests for all the core domain classes
my @schedulers;
ok $schedulers[0] = VRPipe::Scheduler->get(type => 'LSF'), 'created a Scheduler using get()';
is_deeply [$schedulers[0]->id, $schedulers[0]->type], [1, 'LSF'], 'scheduler1 has the expected fields';
ok $schedulers[1] = VRPipe::Scheduler->get(type => 'local'), 'created another Scheduler using get()';
is_deeply [$schedulers[1]->id, $schedulers[1]->type], [2, 'local'], 'scheduler2 has the expected fields';
ok my $default_type = VRPipe::Scheduler->default_type, 'could get a default type';
ok my $default_output_root = VRPipe::Scheduler->default_output_root, 'could get a default output root';
ok $schedulers[2] = VRPipe::Scheduler->get(), 'created another Scheduler using get() with no args';
is_deeply [$schedulers[2]->id, $schedulers[2]->type, $schedulers[2]->output_root], [1, $default_type, $default_output_root], 'scheduler3 has default fields';
$schedulers[2]->start_scheduler;

my $output_dir = dir($schedulers[2]->output_root, 'persistent_test_output');
$schedulers[2]->remove_tree($output_dir);
$schedulers[2]->make_path($output_dir);

my @files;
my $input1_path = file($output_dir, 'input1.txt');
open(my $fh, '>', $input1_path) or die "Could not write to $input1_path\n";
print $fh "input1_line1\ninput2_line2\n";
close($fh);
ok $files[0] = VRPipe::File->get(path => $input1_path, type => 'txt', metadata => {foo => 'bar'}), 'created a File using get()';
my $output1_path = file($output_dir, 'output1.txt');
ok $files[1] = VRPipe::File->get(path => $output1_path, type => 'txt'), 'created another File using get()';

my @ids;
ok $ids[0] = VRPipe::StepIODefinition->get(type => 'bam', description => 'step_1 bam input'), 'created a InputDefinition using get()';

my @steps;
ok $steps[0] = VRPipe::Step->get(name => 'step_1',
                                 inputs_definition => { static_input => $files[0],
                                                        dynamic_input => $ids[0] },
                                 body_sub => sub {
                                                    my $self = shift;
                                                    my $ofile = $self->output_file(output_key => 'step1_output', basename => 'output1.txt', type => 'txt');
                                                    my $fh = $ofile->openw();
                                                    print $fh "step1output\n";
                                                    $ofile->close();
                                                 },
                                 outputs_definition => { step1_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step1_output file') },
                                 post_process_sub => sub { return 1 },
                                 description => 'the first step'), 'created a Step using get()';
is_deeply [$steps[0]->id, $steps[0]->description], [1, 'the first step'], 'step1 has the expected fields';
undef $steps[0];
ok $steps[0] = VRPipe::Step->get(name => 'step_1'), 'got a Step using get(name => )';
is_deeply [$steps[0]->id,
           $steps[0]->description,
           $steps[0]->inputs_definition->{static_input}->path], [1, 'the first step', $input1_path], 'step1 still has the expected fields';

ok my $first_step = VRPipe::Step->get(id => 1), 'step1 could be gotten by id';
is $first_step->description, 'the first step', 'it has the correct description';
is $first_step->description('the 1st step'), 'the first step', 'description does not seem like it was changed prior to an update';
is $steps[0]->description, 'the first step', 'indeed, other instances are not affected either';
$first_step->update;
is $steps[0]->description, 'the 1st step', 'all instances are affected after an update';

my @jobs;
my $epoch_time = time();
ok $jobs[0] = VRPipe::Job->get(cmd => 'echo "job1";'), 'created a Job using get() with no dir';
is_deeply [$jobs[0]->id, $jobs[0]->cmd, $jobs[0]->dir, $jobs[0]->running], [1, 'echo "job1";', cwd(), 0], 'job1 has the expected fields';
undef $jobs[0];
$jobs[0] = VRPipe::Job->get(cmd => 'echo "job1";');
cmp_ok $jobs[0]->creation_time->epoch, '>=', $epoch_time, 'creation time defaulted to just now';
is $jobs[0]->start_time, undef, 'start_time defaults to undef';
$jobs[1] = VRPipe::Job->get(cmd => 'echo "job2";');
$jobs[1]->pid(33);
$jobs[1]->user('foo');
$jobs[1]->host('local');
$jobs[1]->update;
undef $jobs[1];
$jobs[1] = VRPipe::Job->get(id => 2);
is_deeply [$jobs[1]->id, $jobs[1]->pid, $jobs[1]->user, $jobs[1]->host], [2, 33, 'foo', 'local'], 'you can set multiple fields followed by a single update';
$jobs[1]->pid(undef);
$jobs[1]->update;
undef $jobs[1];
$jobs[1] = VRPipe::Job->get(id => 2);
is $jobs[1]->pid, undef, 'you can set a field to undef';

my @reqs;
ok $reqs[0] = VRPipe::Requirements->get(memory => 2000, time => 6), 'created a Requirments using get() with only memory and time';
undef $reqs[0];
$reqs[0] = VRPipe::Requirements->get(memory => 2000, time => 6);
is_deeply [$reqs[0]->id, $reqs[0]->memory, $reqs[0]->time, $reqs[0]->cpus, $reqs[0]->tmp_space, $reqs[0]->local_space, $reqs[0]->custom], [1, 2000, 6, 1, 0, 0, {}], 'reqs1 has the expected fields';
ok $reqs[1] = VRPipe::Requirements->get(memory => 2000, time => 6, cpus => 2, tmp_space => 500, local_space => 0, custom => {loo => 'car'}), 'created a Requirments using fully specified get()';
undef $reqs[1];
$reqs[1] = VRPipe::Requirements->get(memory => 2000, time => 6, cpus => 2, tmp_space => 500, local_space => 0, custom => {loo => 'car'});
is_deeply [$reqs[1]->id, $reqs[1]->memory, $reqs[1]->time, $reqs[1]->cpus, $reqs[1]->tmp_space, $reqs[1]->local_space, $reqs[1]->custom->{loo}], [2, 2000, 6, 2, 500, 0, 'car'], 'reqs2 has the expected fields and is a seperate new entry in the db';

my @ds;
ok $ds[0] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.fivelist'), 'created a DataSource using get()';
undef $ds[0];
$ds[0] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.fivelist');
is_fields [qw/id type method source/], $ds[0], [1, 'list', 'all', 't/data/datasource.fivelist'], 'ds1 has the expected fields';
$ds[1] = VRPipe::DataSource->get(type => 'list', method => 'all', source => 't/data/datasource.onelist');
is_fields [qw/id type method source/], $ds[1], [2, 'list', 'all', 't/data/datasource.onelist'], 'ds2 has the expected fields';

my @de;
foreach my $de_num (1..5) {
    # create
    $ds[0]->next_element;
}
foreach my $de_num (1..5) {
    # get
    push(@de, VRPipe::DataElement->get(id => $de_num));
}
is_deeply [$de[2]->id, $de[2]->datasource->id, $de[2]->result->{line}], [3, 1, 'fed_result_3'], 'de3 has the expected fields';

my @pipelines;
ok $pipelines[0] = VRPipe::Pipeline->get(name => 'p1', description => 'first test pipeline'), 'created a Pipeline using get()';
undef $pipelines[0];
$pipelines[0] = VRPipe::Pipeline->get(name => 'p1', description => 'first test pipeline');
is_fields [qw/id name description/], $pipelines[0], [1, 'p1', 'first test pipeline'], 'pipeline1 has the expected fields';

# create some more steps we can chain together into a proper pipeline
VRPipe::Step->get(name => "step_2",
                  inputs_definition => { step2_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_2 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step2_output', basename => 'step2_output.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step2output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step2_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step2_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_3",
                  inputs_definition => { step3_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_3 input') },
                  body_sub => sub { 
                                    my $self = shift;
                                    my $ofile = $self->output_file(output_key => 'step3_output', basename => 'step3_output.txt', type => 'txt')->path;
                                    $self->dispatch(["sleep 5; echo step3output > $ofile", $self->new_requirements(memory => 50, time => 1)]);
                                    $self->dispatch(["sleep 4;", $self->new_requirements(memory => 50, time => 1)]);
                                    $self->dispatch(["sleep 3;", $self->new_requirements(memory => 50, time => 1)]);
                                   },
                  outputs_definition => { step3_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step3_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_4",
                  inputs_definition => { step4_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_4 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step4_output', basename => 'step4_basename.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step4output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step4_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step4_output file') },
                  post_process_sub => sub { return 1 });
VRPipe::Step->get(name => "step_5",
                  inputs_definition => { step5_input => VRPipe::StepIODefinition->get(type => 'txt', description => 'step_5 input') },
                  body_sub => sub {
                                    my $ofile = shift->output_file(output_key => 'step5_output', basename => 'step5_output.txt', type => 'txt');
                                    my $fh = $ofile->openw();
                                    print $fh "step5output\n";
                                    $ofile->close();
                                   },
                  outputs_definition => { step5_output => VRPipe::StepIODefinition->get(type => 'txt', description => 'step5_output file') },
                  post_process_sub => sub { return 1 });
foreach my $step_num (2..5) {
    # get
    push(@steps, VRPipe::Step->get(id => $step_num));
}
is_deeply [$steps[2]->id, $steps[2]->inputs_definition->{step3_input}->type, $steps[2]->outputs_definition->{step3_output}->description], [3, 'txt', 'step3_output file'], 'step3 has the expected fields';

# create a 5 step pipeline by creating stepmembers
my @stepms;
my $step_num = 0;
foreach my $step (@steps) {
    # create
    VRPipe::StepMember->get(step => $step, pipeline => $pipelines[0], step_number => ++$step_num);
}
for (1..5) {
    # get
    push(@stepms, VRPipe::StepMember->get(id => $_));
}
is_deeply [$stepms[2]->id, $stepms[2]->step->id, $stepms[2]->pipeline->id, $stepms[2]->step_number], [3, 3, 1, 3], 'stepmember3 has the expected fields';

my @setups;
my $step1_bam_input = file($output_dir, 'input.bam');
copy(file(qw(t data file.bam)), $step1_bam_input) || die "Could not copy to $step1_bam_input\n";
ok $setups[0] = VRPipe::PipelineSetup->get(name => 'ps1', datasource => $ds[0], output_root => $output_dir, pipeline => $pipelines[0], options => {dynamic_input => "$step1_bam_input", baz => 'loman'}), 'created a PipelineSetup using get()';
undef $setups[0];
$setups[0] = VRPipe::PipelineSetup->get(id => 1);
is_deeply [$setups[0]->id, $setups[0]->datasource->id, $setups[0]->pipeline->id, $setups[0]->options->{baz}, $setups[0]->options->{dynamic_input}], [1, 1, 1, 'loman', $step1_bam_input], 'pipelinesetup1 has the expected fields when retrievied with just id';
undef $setups[0];
$setups[0] = VRPipe::PipelineSetup->get(name => 'ps1', datasource => $ds[0], output_root => $output_dir, pipeline => $pipelines[0], options => {dynamic_input => "$step1_bam_input", baz => 'loman'});
is_deeply [$setups[0]->id, $setups[0]->datasource->id, $setups[0]->pipeline->id, $setups[0]->options->{baz}], [1, 1, 1, 'loman'], 'pipelinesetup1 has the expected fields when retrieved with a full spec';


ok my $first_setup = VRPipe::PipelineSetup->get(id => 1), 'pipelinesetup1 could be gotten by id';
is $first_setup->datasource->id, 1, 'it has the correct datasource';
$first_setup->datasource($ds[1]); # *** though we shouldn't be able to change an is_key value
$first_setup->update;
is $first_setup->datasource->id, 2, 'datasource could be changed';
is $setups[0]->datasource->id, 2, 'and since we did an update, other instances are affected as well';
$first_setup->datasource($ds[0]);
$first_setup->update;

my @stepstates;
ok $stepstates[0] = VRPipe::StepState->get(stepmember => $stepms[0], dataelement => $de[0], pipelinesetup => $setups[0]), 'created a StepState using get()';
undef $stepstates[0];
$stepstates[0] = VRPipe::StepState->get(id => 1);
is_deeply [$stepstates[0]->id, $stepstates[0]->stepmember->id, $stepstates[0]->dataelement->id, $stepstates[0]->pipelinesetup->id, $stepstates[0]->complete], [1, 1, 1, 1, 0], 'stepstate1 has the expected fields';
$stepstates[1] = VRPipe::StepState->get(stepmember => $stepms[0], dataelement => $de[1], pipelinesetup => $setups[0]);

my @subs;
throws_ok { VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0]) } qr/Attribute \(requirements\) is required/, 'requirements is required when created a Submission, even though it is not a key';
ok $subs[0] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0], requirements => $reqs[0]), 'Submission could be made without required scheduler, which can default';
undef $subs[0];
$subs[0] = VRPipe::Submission->get(id => 1);
is_deeply [$subs[0]->id, $subs[0]->job->cmd, $subs[0]->stepstate->id, $subs[0]->requirements->memory, $subs[0]->retries, $subs[0]->scheduled, $subs[0]->done], [1, 'echo "job1";', 1, 2000, 0, undef, 0], 'submission1 has the expected fields';
ok my $defaulted_scheduler = $subs[0]->scheduler, 'submission1 has a scheduler generated by default';
is $defaulted_scheduler->id, 1, 'The scheduler was the default one';
is $subs[0]->memory, 2000, 'requirement methods pass through to the requriements object';
ok $subs[0]->extra_memory(500), 'could set some extra memory';
is $subs[0]->memory, 2500, 'now our requirements object has more memory';
is $subs[0]->requirements->id, 3, 'and it is actually a new row in the db';
$subs[0]->update;
is $subs[0]->id, 1, 'yet the submission id did not change';
$subs[0]->extra_memory();
is $subs[0]->memory, 3500, 'extra_memory defaults to adding 1000MB';
$subs[1] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[0], requirements => $reqs[1]);
is $subs[1]->id, 1, 'a Submission created with only the reqs differing results in the same submission';
undef $subs[1];
$subs[1] = VRPipe::Submission->get(id => 1);
is $subs[1]->requirements->id, 2, 'but getting it with a certain requirements changed the requirements for this submission in the db';
$subs[1] = VRPipe::Submission->get(job => $jobs[0], stepstate => $stepstates[1], requirements => $reqs[1]);

throws_ok { VRPipe::PersistentArray->get() } qr/needs id or members/, 'get() for PersistentArray fails with no args';
throws_ok { VRPipe::PersistentArray->get(id => 1, members => \@subs) } qr/both id and members cannot be supplied/, 'get() for PersistentArray fails with both id and members supplied';
ok my $subs_array = VRPipe::PersistentArray->get(members => \@subs), 'created a PersistentArray using get(members => [...])';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [1, 1, 2], 'the created PArray has the correct contents';
undef $subs_array;
ok $subs_array = VRPipe::PersistentArray->get(id => 1), 'got a PersistentArray using get(id => 1)';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [1, 1, 2], 'the gotten PArray has the correct contents';
ok $subs_array = VRPipe::PersistentArray->get(members => \@subs), 'created a PersistentArray using the same set of members)';
is_deeply [$subs_array->id, $subs_array->members->[0]->id, $subs_array->members->[1]->id], [2, 1, 2], 'the created PArray has a new id, but the same contents otherwise';
is $subs_array->member(2)->id, 2, 'member() works given an index';
undef $subs_array;
{'fraction_non_alphanumeric': 0.18338956964423134, 'fraction_numerical': 0.026046291363286567, 'mean_word_length': 3.51818668093073, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 831, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 80, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3276694', 'n_tokens_mistral': 11567, 'n_tokens_neox': 11045, 'n_words': 2906}