<commit_before>
=head1 NAME

VRPipe::Schedulers::local - stateless 'scheduler' for the local machine

=head1 SYNOPSIS

*** more documentation to come

=head1 DESCRIPTION

The 'Local' scheduler runs jobs directly on the local CPU on a 'when possible'
basis. It holds no state and no queue, so it is possible for a Submission with
high CPU or memory requirements to 'pend' for ages (or even forever if the
requirements exceed the capability of the local machine). Thus this scheduler
is primarily intended for testing purposes only, though could be used in
production on a powerful enough machine. It is recommended to start the server
with --max_submissions set to 1 less than the number of cores in your machine.

It assumes a standard linux type OS with command-line tools such as 'ps' and
'grep', 'kill' and the /proc filesystem. To get this to work on other operating
systems, make a subclass of this and override the following methods as needed:
_build_find_handler_processes_cmd(), _parse_processes(), _kill_cmd(),
_get_process_runtime_cmd(), _process_runtime_to_seconds(),
_build_get_available_cpus_cmd(), _available_cpus_to_count(),
_get_process_memory_cmd(), _process_memory_to_mb(),
_build_get_available_memory_cmd() and _available_memory_to_mb(). If you OS is
very unusual you might also have to override _run_cmd(). Conceptually, however,
this entire module is predicated on the OS being able to provide an id that
uniquely identifies all the processes associated with a running vrpipe-handler.
If it can't do that, a new class may make more sense than a subclass.

You can also subclass this scheduler for use on a cluster of machines. Just
override _cluster_ips to return a list of ips that can be ssh'd to from the
machine the vrpipe-server will run on. It must be possible to do that without
entering a password. Each machine should have the same environment as the
server machine (ie. same environment variables when logged in, same software,
same shared filesystem mounted etc.). It will be blind to anything else running
on those other machines, so do not do this if the other machines are used for
any other purpose or by anyone else. Instead use or write a scheduler module
that interfaces with a 3rd party scheduling system, like LSF.

=head1 AUTHOR

<NAME> <<EMAIL>>.

=head1 COPYRIGHT AND LICENSE

Copyright (c) 2011-2013 Genome Research Limited.

This file is part of VRPipe.

VRPipe is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program. If not, see L<http://www.gnu.org/licenses/>.

=cut

use VRPipe::Base;

class VRPipe::Schedulers::local with VRPipe::SchedulerMethodsRole {
    use VRPipe::Interface::CmdLine;
    use VRPipe::Persistent::SchemaBase;
    use POSIX qw(ceil);
    
    our $deployment = VRPipe::Persistent::SchemaBase->database_deployment;
    our $backend = VRPipe::Interface::BackEnd->new(deployment => $deployment);
    
    has '_find_handler_processes_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_find_handler_processes_cmd',
    );
    
    has '_get_available_cpus_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_get_available_cpus_cmd',
    );
    
    has '_get_available_memory_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_get_available_memory_cmd',
    );
    
    sub _build_find_handler_processes_cmd {
        return 'ps xo pid,pgid,command | grep vrpipe-handler';
    }
    
    sub _build_get_available_cpus_cmd {
        return qq{perl -MSys::CPU -e 'print Sys::CPU::cpu_count, qq[\n];'};
    }
    
    sub _build_get_available_memory_cmd {
        return 'free -m';
    }
    
    method submit_command (VRPipe::Requirements :$requirements!, Str|File :$stdo_file!, Str|File :$stde_file!, Str :$cmd!, PositiveInt :$count = 1, Str :$cwd?) {
        # we call a method in this module to submit
        my $class  = ref($self);
        my ($type) = $class =~ /VRPipe::Schedulers::(\S+)/;
        my $exe    = VRPipe::Interface::CmdLine->vrpipe_perl_e("VRPipe::Scheduler->get(type => q[$type])->scheduler_instance->submit(\@ARGV)", $deployment);
        
        # access the requirements object and build up the string based on
        # memory and cpu (other reqs do not apply)
        my $queue              = $self->determine_queue($requirements);
        my $megabytes          = $requirements->memory;
        my $requirments_string = "queue $queue memory $megabytes";
        my $cpus               = $requirements->cpus;
        if ($cpus > 1) {
            $requirments_string .= " cpus $cpus";
        }
        if ($cwd) {
            $requirments_string .= " cwd $cwd";
        }
        
        return $exe . qq[$requirments_string count $count cmd '$cmd'];
    }
    
    sub submit {
        my ($self, %args) = @_;
        my $queue     = $args{queue}  || $self->throw("No queue supplied");
        my $megabytes = $args{memory} || $self->throw("No memory supplied");
        my $count     = $args{count}  || $self->throw("No count supplied");
        my $cmd       = $args{cmd}    || $self->throw("No cmd supplied");
        my $cpus      = $args{cpus}   || 1;
        my $cwd       = $args{cwd};
        <commit_msg>fixed submit_command by adding a space between exe and requirements string<commit_after>
=head1 NAME

VRPipe::Schedulers::local - stateless 'scheduler' for the local machine

=head1 SYNOPSIS

*** more documentation to come

=head1 DESCRIPTION

The 'Local' scheduler runs jobs directly on the local CPU on a 'when possible'
basis. It holds no state and no queue, so it is possible for a Submission with
high CPU or memory requirements to 'pend' for ages (or even forever if the
requirements exceed the capability of the local machine). Thus this scheduler
is primarily intended for testing purposes only, though could be used in
production on a powerful enough machine. It is recommended to start the server
with --max_submissions set to 1 less than the number of cores in your machine.

It assumes a standard linux type OS with command-line tools such as 'ps' and
'grep', 'kill' and the /proc filesystem. To get this to work on other operating
systems, make a subclass of this and override the following methods as needed:
_build_find_handler_processes_cmd(), _parse_processes(), _kill_cmd(),
_get_process_runtime_cmd(), _process_runtime_to_seconds(),
_build_get_available_cpus_cmd(), _available_cpus_to_count(),
_get_process_memory_cmd(), _process_memory_to_mb(),
_build_get_available_memory_cmd() and _available_memory_to_mb(). If you OS is
very unusual you might also have to override _run_cmd(). Conceptually, however,
this entire module is predicated on the OS being able to provide an id that
uniquely identifies all the processes associated with a running vrpipe-handler.
If it can't do that, a new class may make more sense than a subclass.

You can also subclass this scheduler for use on a cluster of machines. Just
override _cluster_ips to return a list of ips that can be ssh'd to from the
machine the vrpipe-server will run on. It must be possible to do that without
entering a password. Each machine should have the same environment as the
server machine (ie. same environment variables when logged in, same software,
same shared filesystem mounted etc.). It will be blind to anything else running
on those other machines, so do not do this if the other machines are used for
any other purpose or by anyone else. Instead use or write a scheduler module
that interfaces with a 3rd party scheduling system, like LSF.

=head1 AUTHOR

<NAME> <<EMAIL>>.

=head1 COPYRIGHT AND LICENSE

Copyright (c) 2011-2013 Genome Research Limited.

This file is part of VRPipe.

VRPipe is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program. If not, see L<http://www.gnu.org/licenses/>.

=cut

use VRPipe::Base;

class VRPipe::Schedulers::local with VRPipe::SchedulerMethodsRole {
    use VRPipe::Interface::CmdLine;
    use VRPipe::Persistent::SchemaBase;
    use POSIX qw(ceil);
    
    our $deployment = VRPipe::Persistent::SchemaBase->database_deployment;
    our $backend = VRPipe::Interface::BackEnd->new(deployment => $deployment);
    
    has '_find_handler_processes_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_find_handler_processes_cmd',
    );
    
    has '_get_available_cpus_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_get_available_cpus_cmd',
    );
    
    has '_get_available_memory_cmd' => (
        is      => 'ro',
        isa     => 'Str',
        lazy    => 1,
        builder => '_build_get_available_memory_cmd',
    );
    
    sub _build_find_handler_processes_cmd {
        return 'ps xo pid,pgid,command | grep vrpipe-handler';
    }
    
    sub _build_get_available_cpus_cmd {
        return qq{perl -MSys::CPU -e 'print Sys::CPU::cpu_count, qq[\n];'};
    }
    
    sub _build_get_available_memory_cmd {
        return 'free -m';
    }
    
    method submit_command (VRPipe::Requirements :$requirements!, Str|File :$stdo_file!, Str|File :$stde_file!, Str :$cmd!, PositiveInt :$count = 1, Str :$cwd?) {
        # we call a method in this module to submit
        my $class  = ref($self);
        my ($type) = $class =~ /VRPipe::Schedulers::(\S+)/;
        my $exe    = VRPipe::Interface::CmdLine->vrpipe_perl_e("VRPipe::Scheduler->get(type => q[$type])->scheduler_instance->submit(\@ARGV)", $deployment);
        
        # access the requirements object and build up the string based on
        # memory and cpu (other reqs do not apply)
        my $queue              = $self->determine_queue($requirements);
        my $megabytes          = $requirements->memory;
        my $requirments_string = "queue $queue memory $megabytes";
        my $cpus               = $requirements->cpus;
        if ($cpus > 1) {
            $requirments_string .= " cpus $cpus";
        }
        if ($cwd) {
            $requirments_string .= " cwd $cwd";
        }
        
        return $exe . qq[ $requirments_string count $count cmd '$cmd'];
    }
    
    sub submit {
        my ($self, %args) = @_;
        my $queue     = $args{queue}  || $self->throw("No queue supplied");
        my $megabytes = $args{memory} || $self->throw("No memory supplied");
        my $count     = $args{count}  || $self->throw("No count supplied");
        my $cmd       = $args{cmd}    || $self->throw("No cmd supplied");
        my $cpus      = $args{cpus}   || 1;
        my $cwd       = $args{cwd};
        
{'fraction_non_alphanumeric': 0.09222298692360634, 'fraction_numerical': 0.003785271851342051, 'mean_word_length': 3.1666666666666665, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3499087', 'n_tokens_mistral': 3457, 'n_tokens_neox': 3348, 'n_words': 1319}