<commit_before>package GRNOC::Simp::Data;

use strict;
use warnings;
use Moo;
use Types::Standard qw( Str Bool );

use Parallel::ForkManager;
use Proc::Daemon;

use GRNOC::Config;
use GRNOC::Log;

use GRNOC::Simp::Data::Worker;

### required attributes ###

has config_file => ( is => 'ro',
                     isa => Str,
                     required => 1 );

has logging_file => ( is => 'ro',
                      isa => Str,
                      required => 1 );

### optional attributes ###

has daemonize => ( is => 'ro',
                   isa => Bool,
                   default => 1 );

### private attributes ###

has config => ( is => 'rwp' );

has logger => ( is => 'rwp' );

has children => ( is => 'rwp',
                  default => sub { [] } );

sub BUILD{
    my ($self) = @_;

    # create and store logger object
    my $grnoc_log = GRNOC::Log->new( config => $self->logging_file );
    my $logger = GRNOC::Log->get_logger();

    $self->_set_logger( $logger );

    # create and store config object
    my $config = GRNOC::Config->new( config_file => $self->config_file,
                                     force_array => 0 );

    $self->_set_config( $config );

    return $self;
}


sub start {

    my ( $self ) = @_;

    $self->logger->info( 'Starting.' );

    $self->logger->debug( 'Setting up signal handlers.' );

    # setup signal handlers
    $SIG{'TERM'} = sub {

        $self->logger->info( 'Received SIG TERM.' );
        $self->stop();
    };

    $SIG{'HUP'} = sub {

        $self->logger->info( 'Received SIG HUP.' );
    };

    # need to daemonize
    if ( $self->daemonize ) {


        $self->logger->debug( 'Daemonizing.' );

        my $daemon = Proc::Daemon->new( pid_file => $self->config->get( '/config/pid-file' ) );

        my $pid = $daemon->Init();

        # in child/daemon process
        if ( !$pid ) {

            $self->logger->debug( 'Created daemon process.' );

            # change process name
            $0 = "simpPoller";

            $self->_create_workers();
        }
    }

    # dont need to daemonize
    else {

        $self->logger->debug( 'Running in foreground.' );

        #-- when in fg just act as a working directly with no sub processes so we can nytprof 
        my $worker = GRNOC::Simp::Data::Worker->new( config    => $self->config,
						     logger    => $self->logger,
						     worker_id => 13 );
	
        # this should only return if we tell it to stop via TERM signal etc.
        $worker->start();
    }

    return 1;
}

sub stop {

    my ( $self ) = @_;

    $self->logger->info( 'Stopping.' );

    my @pids = @{$self->children};

    $self->logger->debug( 'Stopping child worker processes ' . join( ' ', @pids ) . '.' );

    return kill( 'TERM', @pids );
}

#-------- end of multprocess boilerplate
sub _create_workers {

    my ( $self ) = @_;

    my $workers = $self->config->get( '/config/@workers' );

    $self->logger->info( "Creating $workers child worker processes." );

    my $forker = Parallel::ForkManager->new( $workers );

    # keep track of children pids
    $forker->run_on_start( sub {

        my ( $pid ) = @_;
        $self->logger->debug( "Child worker process $pid created." );
        push( @{$self->children}, $pid );
                           } );
    # create high res workers
    for (my $worker_id=0; $worker_id<$workers;$worker_id++) {
        
        $forker->start() and next;


	# create worker in this process
	my $worker = GRNOC::Simp::Data::Worker->new( config    => $self->config,
						     logger    => $self->logger,
						     worker_id => $worker_id );
	
	# this should only return if we tell it to stop via TERM signal etc.
	$worker->start();
	
	# exit child process
        $forker->finish();
    }

    $self->logger->debug( 'Waiting for all child worker processes to exit.' );

    # wait for all children to return
    $forker->wait_all_children();

    $self->_set_children( [] );

    $self->logger->debug( 'All child workers have exited.' );
}

1;
<commit_msg>fixing small issue with how we are setting process name<commit_after>package GRNOC::Simp::Data;

use strict;
use warnings;
use Moo;
use Types::Standard qw( Str Bool );

use Parallel::ForkManager;
use Proc::Daemon;

use GRNOC::Config;
use GRNOC::Log;

use GRNOC::Simp::Data::Worker;

### required attributes ###

has config_file => ( is => 'ro',
                     isa => Str,
                     required => 1 );

has logging_file => ( is => 'ro',
                      isa => Str,
                      required => 1 );

### optional attributes ###

has daemonize => ( is => 'ro',
                   isa => Bool,
                   default => 1 );

### private attributes ###

has config => ( is => 'rwp' );

has logger => ( is => 'rwp' );

has children => ( is => 'rwp',
                  default => sub { [] } );

sub BUILD{
    my ($self) = @_;

    # create and store logger object
    my $grnoc_log = GRNOC::Log->new( config => $self->logging_file );
    my $logger = GRNOC::Log->get_logger();

    $self->_set_logger( $logger );

    # create and store config object
    my $config = GRNOC::Config->new( config_file => $self->config_file,
                                     force_array => 0 );

    $self->_set_config( $config );

    return $self;
}


sub start {

    my ( $self ) = @_;

    $self->logger->info( 'Starting.' );

    $self->logger->debug( 'Setting up signal handlers.' );

    # setup signal handlers
    $SIG{'TERM'} = sub {

        $self->logger->info( 'Received SIG TERM.' );
        $self->stop();
    };

    $SIG{'HUP'} = sub {

        $self->logger->info( 'Received SIG HUP.' );
    };

    # need to daemonize
    if ( $self->daemonize ) {


        $self->logger->debug( 'Daemonizing.' );

        my $daemon = Proc::Daemon->new( pid_file => $self->config->get( '/config/pid-file' ) );

        my $pid = $daemon->Init();

        # in child/daemon process
        if ( !$pid ) {

            $self->logger->debug( 'Created daemon process.' );

            # change process name
            $0 = "SimpData";

            $self->_create_workers();
        }
    }

    # dont need to daemonize
    else {

        $self->logger->debug( 'Running in foreground.' );

        #-- when in fg just act as a working directly with no sub processes so we can nytprof 
        my $worker = GRNOC::Simp::Data::Worker->new( config    => $self->config,
						     logger    => $self->logger,
						     worker_id => 13 );
	
        # this should only return if we tell it to stop via TERM signal etc.
        $worker->start();
    }

    return 1;
}

sub stop {

    my ( $self ) = @_;

    $self->logger->info( 'Stopping.' );

    my @pids = @{$self->children};

    $self->logger->debug( 'Stopping child worker processes ' . join( ' ', @pids ) . '.' );

    return kill( 'TERM', @pids );
}

#-------- end of multprocess boilerplate
sub _create_workers {

    my ( $self ) = @_;

    my $workers = $self->config->get( '/config/@workers' );

    $self->logger->info( "Creating $workers child worker processes." );

    my $forker = Parallel::ForkManager->new( $workers );

    # keep track of children pids
    $forker->run_on_start( sub {

        my ( $pid ) = @_;
        $self->logger->debug( "Child worker process $pid created." );
        push( @{$self->children}, $pid );
                           } );
    # create high res workers
    for (my $worker_id=0; $worker_id<$workers;$worker_id++) {
        
        $forker->start() and next;


	# create worker in this process
	my $worker = GRNOC::Simp::Data::Worker->new( config    => $self->config,
						     logger    => $self->logger,
						     worker_id => $worker_id );
	
	# this should only return if we tell it to stop via TERM signal etc.
	$worker->start();
	
	# exit child process
        $forker->finish();
    }

    $self->logger->debug( 'Waiting for all child worker processes to exit.' );

    # wait for all children to return
    $forker->wait_all_children();

    $self->_set_children( [] );

    $self->logger->debug( 'All child workers have exited.' );
}

1;

{'fraction_non_alphanumeric': 0.1632047477744807, 'fraction_numerical': 0.002472799208704253, 'mean_word_length': 2.087404580152672, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 177, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5734398', 'n_tokens_mistral': 2691, 'n_tokens_neox': 2471, 'n_words': 689}