<commit_before>package GRNOC::TSDS::Writer::Worker;

use Moo;
use Types::Standard qw( Str Int HashRef Object Maybe );

use GRNOC::TSDS::DataType;
use GRNOC::TSDS::Constants;
use GRNOC::TSDS::AggregateDocument;
use GRNOC::TSDS::DataDocument;
use GRNOC::TSDS::EventDocument;
use GRNOC::TSDS::Writer::AggregateMessage;
use GRNOC::TSDS::Writer::DataMessage;
use GRNOC::TSDS::Writer::EventMessage;
use GRNOC::TSDS::RedisLock;

use MongoDB;
use Net::AMQP::RabbitMQ;
use Cache::Memcached::Fast;
use Tie::IxHash;
use JSON::XS;
use Math::Round qw( nlowmult );
use Time::HiRes qw( time );
use Try::Tiny;

use Data::Dumper;

### constants ###

use constant DATA_CACHE_EXPIRATION => 60 * 60;
use constant AGGREGATE_CACHE_EXPIRATION => 60 * 60;
use constant MEASUREMENT_CACHE_EXPIRATION => 60 * 60;
use constant QUEUE_PREFETCH_COUNT => 20;
use constant QUEUE_FETCH_TIMEOUT => 10 * 1000;
use constant RECONNECT_TIMEOUT => 10;

### required attributes ###

has config => ( is => 'ro',
                required => 1 );

has logger => ( is => 'ro',<commit_msg>reducing queue prefetch count, trying a couple of things to see what impact to memory is when things slow down<commit_after>package GRNOC::TSDS::Writer::Worker;

use Moo;
use Types::Standard qw( Str Int HashRef Object Maybe );

use GRNOC::TSDS::DataType;
use GRNOC::TSDS::Constants;
use GRNOC::TSDS::AggregateDocument;
use GRNOC::TSDS::DataDocument;
use GRNOC::TSDS::EventDocument;
use GRNOC::TSDS::Writer::AggregateMessage;
use GRNOC::TSDS::Writer::DataMessage;
use GRNOC::TSDS::Writer::EventMessage;
use GRNOC::TSDS::RedisLock;

use MongoDB;
use Net::AMQP::RabbitMQ;
use Cache::Memcached::Fast;
use Tie::IxHash;
use JSON::XS;
use Math::Round qw( nlowmult );
use Time::HiRes qw( time );
use Try::Tiny;

use Data::Dumper;

### constants ###

use constant DATA_CACHE_EXPIRATION => 60 * 60;
use constant AGGREGATE_CACHE_EXPIRATION => 60 * 60;
use constant MEASUREMENT_CACHE_EXPIRATION => 60 * 60;
use constant QUEUE_PREFETCH_COUNT => 5;
use constant QUEUE_FETCH_TIMEOUT => 10 * 1000;
use constant RECONNECT_TIMEOUT => 10;

### required attributes ###

has config => ( is => 'ro',
                required => 1 );

has logger => ( is => 'ro',
{'fraction_non_alphanumeric': 0.14161319890009166, 'fraction_numerical': 0.020623281393217233, 'mean_word_length': 6.180921052631579, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4488966', 'n_tokens_mistral': 846, 'n_tokens_neox': 792, 'n_words': 204}