<commit_before>package Minion::Backend::Pg;
use Mojo::Base 'Minion::Backend';

use Mojo::IOLoop;
use Mojo::Pg;
use Sys::Hostname 'hostname';

has 'pg';

sub dequeue {
  my ($self, $id, $timeout) = @_;

  if ((my $job = $self->_try($id)) || Mojo::IOLoop->is_running) { return $job }

  my $db = $self->pg->db;
  $db->listen('minion.job')->on(notification => sub { Mojo::IOLoop->stop });
  my $timer = Mojo::IOLoop->timer($timeout => sub { Mojo::IOLoop->stop });
  Mojo::IOLoop->start;
  $db->unlisten('*') and Mojo::IOLoop->remove($timer);
  undef $db;

  return $self->_try($id);
}

sub enqueue {
  my ($self, $task) = (shift, shift);
  my $args    = shift // [];
  my $options = shift // {};

  my $db = $self->pg->db;
  return $db->query(
    "insert into minion_jobs (args, delayed, priority, task)
     values (?, (now() + (interval '1 second' * ?)), ?, ?)
     returning id", {json => $args}, $options->{delay} // 0,
    $options->{priority} // 0, $task
  )->hash->{id};
}

sub fail_job   { shift->_update(1, @_) }
sub finish_job { shift->_update(0, @_) }

sub job_info {
  shift->pg->db->query(
    'select id, args, extract(epoch from created) as created,
       extract(epoch from delayed) as delayed,
       extract(epoch from finished) as finished, priority, result,
       extract(epoch from retried) as retried, retries,
       extract(epoch from started) as started, state, task, worker
     from minion_jobs where id = ?', shift
  )->expand->hash;
}

sub list_jobs {
  my ($self, $offset, $limit, $options) = @_;

  return $self->pg->db->query(
    'select id
     from minion_jobs
     where (state = $1 or $1 is null) and (task = $2 or $2 is null)
     order by id desc
     limit $3
     offset $4', @$options{qw(state task)}, $limit, $offset
  )->arrays->map(sub { $self->job_info($_->[0]) })->to_array;
}

sub list_workers {
  my ($self, $offset, $limit) = @_;

  my $sql = 'select id from minion_workers order by id desc limit ? offset ?';
  return $self->pg->db->query($sql, $limit, $offset)
    ->arrays->map(sub { $self->worker_info($_->[0]) })->to_array;
}

sub new {
  my $self = shift->SUPER::new(pg => Mojo::Pg->new(@_));
  my $pg = $self->pg->max_connections(1);
  $pg->migrations->name('minion')->from_data;
  $pg->once(connection => sub { shift->migrations->migrate });
  return $self;
}

sub register_worker {
  my ($self, $id) = @_;

  my $sql
    = 'update minion_workers set notified = now() where id = ? returning 1';
  return $id if $id && $self->pg->db->query($sql, $id)->rows;

  $sql = 'insert into minion_workers (host, pid) values (?, ?) returning id';
  return $self->pg->db->query($sql, hostname, $$)->hash->{id};
}

sub remove_job {
  !!shift->pg->db->query(
    "delete from minion_jobs
     where id = ? and state in ('inactive', 'failed', 'finished')
     returning 1", shift
  )->rows;
}

sub repair {
  my $self = shift;

  # Check worker registry
  my $db     = $self->pg->db;
  my $minion = $self->minion;
  $db->query(
    "delete from minion_workers
     where notified < now() - interval '1 second' * ?", $minion->missing_after
  );

  # Abandoned jobs
  $db->query(
    "update minion_jobs as j
     set result = to_json('Worker went away'::text), state = 'failed'
     where state = 'active'
       and not exists(select 1 from minion_workers where id = j.worker)"
  );

  # Old jobs
  $db->query(
    "delete from minion_jobs
     where state = 'finished' and finished < now() - interval '1 second' * ?",
    $minion->remove_after
  );
}

sub reset {
  my $db = shift->pg->db;
  $db->query('delete from minion_jobs');
  $db->query('delete from minion_workers');
}

sub retry_job {
  !!shift->pg->db->query(
    "update minion_jobs
     set finished = null, result = null, retried = now(),
       retries = retries + 1, started = null, state = 'inactive',
       worker = null
     where id = ? and state in ('failed', 'finished')
     returning 1", shift
  )->rows;
}

sub stats {
  my $self = shift;

  my $db  = $self->pg->db;
  my $all = $db->query('select count(*) from minion_workers')->array->[0];
  my $sql
    = "select count(distinct worker) from minion_jobs where state = 'active'";
  my $active = $db->query($sql)->array->[0];

  $sql = 'select state, count(state) from minion_jobs group by 1';
  my $states = $db->query($sql)
    ->arrays->reduce(sub { $a->{$b->[0]} = $b->[1]; $a }, {});

  return {
    active_workers   => $active,
    inactive_workers => $all - $active,
    active_jobs      => $states->{active} || 0,
    inactive_jobs    => $states->{inactive} || 0,
    failed_jobs      => $states->{failed} || 0,
    finished_jobs    => $states->{finished} || 0,
  };
}

sub unregister_worker {
  shift->pg->db->query('delete from minion_workers where id = ?', shift);
}

sub worker_info {
  shift->pg->db->query(
    "select id, extract(epoch from notified) as notified, array(
       select id from minion_jobs
         where worker = minion_workers.id and state = 'active'
     ) as jobs, host, pid, extract(epoch from started) as started
     from minion_workers
     where id = ?", shift
  )->hash;
}

sub _try {
  my ($self, $id) = @_;

  return $self->pg->db->query(
    "update minion_jobs
     set started = now(), state = 'active', worker = ?
     from (
       select id
       from minion_jobs
       where state = 'inactive' and delayed < now() and task = any (?)
       order by priority desc, created<commit_msg>more consistent SQL formatting<commit_after>package Minion::Backend::Pg;
use Mojo::Base 'Minion::Backend';

use Mojo::IOLoop;
use Mojo::Pg;
use Sys::Hostname 'hostname';

has 'pg';

sub dequeue {
  my ($self, $id, $timeout) = @_;

  if ((my $job = $self->_try($id)) || Mojo::IOLoop->is_running) { return $job }

  my $db = $self->pg->db;
  $db->listen('minion.job')->on(notification => sub { Mojo::IOLoop->stop });
  my $timer = Mojo::IOLoop->timer($timeout => sub { Mojo::IOLoop->stop });
  Mojo::IOLoop->start;
  $db->unlisten('*') and Mojo::IOLoop->remove($timer);
  undef $db;

  return $self->_try($id);
}

sub enqueue {
  my ($self, $task) = (shift, shift);
  my $args    = shift // [];
  my $options = shift // {};

  my $db = $self->pg->db;
  return $db->query(
    "insert into minion_jobs (args, delayed, priority, task)
     values (?, (now() + (interval '1 second' * ?)), ?, ?)
     returning id", {json => $args}, $options->{delay} // 0,
    $options->{priority} // 0, $task
  )->hash->{id};
}

sub fail_job   { shift->_update(1, @_) }
sub finish_job { shift->_update(0, @_) }

sub job_info {
  shift->pg->db->query(
    'select id, args, extract(epoch from created) as created,
       extract(epoch from delayed) as delayed,
       extract(epoch from finished) as finished, priority, result,
       extract(epoch from retried) as retried, retries,
       extract(epoch from started) as started, state, task, worker
     from minion_jobs where id = ?', shift
  )->expand->hash;
}

sub list_jobs {
  my ($self, $offset, $limit, $options) = @_;

  return $self->pg->db->query(
    'select id from minion_jobs
     where (state = $1 or $1 is null) and (task = $2 or $2 is null)
     order by id desc
     limit $3
     offset $4', @$options{qw(state task)}, $limit, $offset
  )->arrays->map(sub { $self->job_info($_->[0]) })->to_array;
}

sub list_workers {
  my ($self, $offset, $limit) = @_;

  my $sql = 'select id from minion_workers order by id desc limit ? offset ?';
  return $self->pg->db->query($sql, $limit, $offset)
    ->arrays->map(sub { $self->worker_info($_->[0]) })->to_array;
}

sub new {
  my $self = shift->SUPER::new(pg => Mojo::Pg->new(@_));
  my $pg = $self->pg->max_connections(1);
  $pg->migrations->name('minion')->from_data;
  $pg->once(connection => sub { shift->migrations->migrate });
  return $self;
}

sub register_worker {
  my ($self, $id) = @_;

  my $sql
    = 'update minion_workers set notified = now() where id = ? returning 1';
  return $id if $id && $self->pg->db->query($sql, $id)->rows;

  $sql = 'insert into minion_workers (host, pid) values (?, ?) returning id';
  return $self->pg->db->query($sql, hostname, $$)->hash->{id};
}

sub remove_job {
  !!shift->pg->db->query(
    "delete from minion_jobs
     where id = ? and state in ('inactive', 'failed', 'finished')
     returning 1", shift
  )->rows;
}

sub repair {
  my $self = shift;

  # Check worker registry
  my $db     = $self->pg->db;
  my $minion = $self->minion;
  $db->query(
    "delete from minion_workers
     where notified < now() - interval '1 second' * ?", $minion->missing_after
  );

  # Abandoned jobs
  $db->query(
    "update minion_jobs as j
     set result = to_json('Worker went away'::text), state = 'failed'
     where state = 'active'
       and not exists(select 1 from minion_workers where id = j.worker)"
  );

  # Old jobs
  $db->query(
    "delete from minion_jobs
     where state = 'finished' and finished < now() - interval '1 second' * ?",
    $minion->remove_after
  );
}

sub reset {
  my $db = shift->pg->db;
  $db->query('delete from minion_jobs');
  $db->query('delete from minion_workers');
}

sub retry_job {
  !!shift->pg->db->query(
    "update minion_jobs
     set finished = null, result = null, retried = now(),
       retries = retries + 1, started = null, state = 'inactive',
       worker = null
     where id = ? and state in ('failed', 'finished')
     returning 1", shift
  )->rows;
}

sub stats {
  my $self = shift;

  my $db  = $self->pg->db;
  my $all = $db->query('select count(*) from minion_workers')->array->[0];
  my $sql
    = "select count(distinct worker) from minion_jobs where state = 'active'";
  my $active = $db->query($sql)->array->[0];

  $sql = 'select state, count(state) from minion_jobs group by 1';
  my $states = $db->query($sql)
    ->arrays->reduce(sub { $a->{$b->[0]} = $b->[1]; $a }, {});

  return {
    active_workers   => $active,
    inactive_workers => $all - $active,
    active_jobs      => $states->{active} || 0,
    inactive_jobs    => $states->{inactive} || 0,
    failed_jobs      => $states->{failed} || 0,
    finished_jobs    => $states->{finished} || 0,
  };
}

sub unregister_worker {
  shift->pg->db->query('delete from minion_workers where id = ?', shift);
}

sub worker_info {
  shift->pg->db->query(
    "select id, extract(epoch from notified) as notified, array(
       select id from minion_jobs
         where worker = minion_workers.id and state = 'active'
     ) as jobs, host, pid, extract(epoch from started) as started
     from minion_workers
     where id = ?", shift
  )->hash;
}

sub _try {
  my ($self, $id) = @_;

  return $self->pg->db->query(
    "update minion_jobs
     set started = now(), state = 'active', worker = ?
     from (
       select id from minion_jobs
       where state = 'inactive' and delayed < now() and task = any (?)
       order by priority desc, created
{'fraction_non_alphanumeric': 0.18203681435574878, 'fraction_numerical': 0.005549902876699658, 'mean_word_length': 3.1745173745173747, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 245, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4705251', 'n_tokens_mistral': 3877, 'n_tokens_neox': 3785, 'n_words': 1040}