<commit_before>package WWW::Livesite::Responders::Hub;
use strict;
use Encode;
use Error qw(:try);
use LWP::UserAgent;
use Fcntl qw(:flock);
use Perl::Module;
use Perl::Util qw(:const);
use Error::Programatic;
use Error::Logical;
use Data::Hub qw($Hub);
use Data::Hub::Util qw(:all);
use base qw(WWW::Livesite::Responders::Base);

our $VERSION = 0.1;
our %Commands = ();

# ------------------------------------------------------------------------------
# new - Construcotr
# new $uri, $res
#
# This responder conforms to the /api namespace specification.  Where the URI
#
#   /api/hub/fetch
# 
# indicates a request to invoke 'fetch' on the 'hub' (this) responder.
#
# Parameters:
#
#   target       Target address
# ------------------------------------------------------------------------------

sub new {
  my $pkg = ref($_[0]) ? ref(shift) : shift;
  my $self = $pkg->SUPER::new(@_);
  my @addr = addr_split($$self{'uri'});
  $self->{'verb'} = $addr[2]; # current command (stateful)
  $self->{'params'} = undef; # input parameters (required)
  $self->{'result'} = undef; # output location (stateful)
  $$Hub{'/sys/log'}->debug(sprintf('HUB: uri=%s, verb=%s', $$self{uri}, $$self{verb}));
  $self;
}

sub get_permission_mode {
  my $self = shift;
  'x'; # must have execute permission to use this responder
}

sub can_upload {
  my $self = shift;
  $self->{'verb'} eq 'upload';
}

# If the responder can handle uploads for this request, insert our progress
# meter (input filter).
#  - When using a reverse-proxy like Nginx, uploads are spooled.
#  - libapreq2 has a mechanism for this, is it worth migrating to?
sub get_input_filter {
  my $self = shift;
  return $self->{'verb'} eq 'upload'
    ? __PACKAGE__ . '->rif_upload'
    : undef;
}

sub max_post_size {
  my $self = shift;
  # TODO - Glean values from configuration
  $self->{'verb'} eq 'upload' ? ONE_MB * 500 : ONE_MB * 2;
}

# ------------------------------------------------------------------------------
# rif_upload - Request Input Filter for Uploads (record bytes read)
# See also: Apache2::UploadStatus
#
# Server configuration:
#
#   /sys/conf/debug/slow_upload => .3
#
#     Seconds (via sleep) that the input filter waits while reading the request
#     body. This slows down the reading of uploaded data. Used to debug 
#     client-side upload progress indicators.
#
#     Note this doesn't work behind a reverse proxy (such as NGiNX) as it 
#     handles the client upload.
# ------------------------------------------------------------------------------

sub rif_upload {
  my ($class, $f, $bb, $mode, $block, $readbytes) = @_;
  $$Hub{'/sys/log'}->set_request($f->r);
  my $xfrid = $Hub->{'sys'}{'response'}->get('tmp/xfrid');
  if (!$xfrid) {
    $$Hub{'/sys/log'}->debug('TRANSFER: File-upload was not initialized, progress reporting disabled');
    return Apache2::Const::DECLINED;
  }
  my $ctx = $f->ctx || {
    state => 'uploading',
    mtime => 0,
    size => $f->r->headers_in->get('Content-Length') || 0,
    received => 0,
    slow_upload => $$Hub{'/sys/conf/debug/slow_upload'},
  };
  my $status = $f->next->get_brigade($bb, $mode, $block, $readbytes);
  return $status unless $status == APR::Const::SUCCESS;
  $ctx->{received} += $bb->length;
  # Do not write stats more than once per second
  if (!$ctx->{'mtime'} || (time - $ctx->{'mtime'}) > 1
      || $$ctx{'received'} == $$ctx{'size'}) {
    $ctx->{'mtime'} = time;
    $Hub->{'sys'}{'tmp'}->set("xfr/$xfrid.hf", $ctx)->save();
  }
  $f->ctx($ctx);
  if ($$ctx{'slow_upload'}) {
    $$Hub{'/sys/log'}->warn(sprintf('TRANSFER: [%s] received %d of %d',
      $xfrid, $$ctx{'received'}, $$ctx{'size'}));
    sleep(.3);
  }
  return Apache2::Const::OK;
}

sub compile {
  my $self = shift;
  $Hub->{'/sys/request'}->assert_same_origin();
#
# TODO - This requires all modules which use hub.js to include the auth-token
#
# $Hub->{'/sys/request'}->assert_auth_token();
  my $cgi = $Hub->get('/sys/request/cgi') or die;
  my $resp = $Hub->get('/sys/response') or die;
  $self->run($cgi, $resp);
  $resp->{'headers'}{'Cache-Control'} = 'no-cache, no-store';
  $resp->format;
}

sub run {
  my $self = shift;
  my $verb = $self->{'verb'};
  my $sub = $Commands{$verb} or throw Error::Logical('No such command');
  $self->{'params'} = shift;
  $self->{'result'} = shift;
  $self->{'result'}->set('/head/verb', $verb);
  try {<commit_msg>Removed hard-coded slow_upload sleep interval<commit_after>package WWW::Livesite::Responders::Hub;
use strict;
use Encode;
use Error qw(:try);
use LWP::UserAgent;
use Fcntl qw(:flock);
use Perl::Module;
use Perl::Util qw(:const);
use Error::Programatic;
use Error::Logical;
use Data::Hub qw($Hub);
use Data::Hub::Util qw(:all);
use base qw(WWW::Livesite::Responders::Base);

our $VERSION = 0.1;
our %Commands = ();

# ------------------------------------------------------------------------------
# new - Construcotr
# new $uri, $res
#
# This responder conforms to the /api namespace specification.  Where the URI
#
#   /api/hub/fetch
# 
# indicates a request to invoke 'fetch' on the 'hub' (this) responder.
#
# Parameters:
#
#   target       Target address
# ------------------------------------------------------------------------------

sub new {
  my $pkg = ref($_[0]) ? ref(shift) : shift;
  my $self = $pkg->SUPER::new(@_);
  my @addr = addr_split($$self{'uri'});
  $self->{'verb'} = $addr[2]; # current command (stateful)
  $self->{'params'} = undef; # input parameters (required)
  $self->{'result'} = undef; # output location (stateful)
  $$Hub{'/sys/log'}->debug(sprintf('HUB: uri=%s, verb=%s', $$self{uri}, $$self{verb}));
  $self;
}

sub get_permission_mode {
  my $self = shift;
  'x'; # must have execute permission to use this responder
}

sub can_upload {
  my $self = shift;
  $self->{'verb'} eq 'upload';
}

# If the responder can handle uploads for this request, insert our progress
# meter (input filter).
#  - When using a reverse-proxy like Nginx, uploads are spooled.
#  - libapreq2 has a mechanism for this, is it worth migrating to?
sub get_input_filter {
  my $self = shift;
  return $self->{'verb'} eq 'upload'
    ? __PACKAGE__ . '->rif_upload'
    : undef;
}

sub max_post_size {
  my $self = shift;
  # TODO - Glean values from configuration
  $self->{'verb'} eq 'upload' ? ONE_MB * 500 : ONE_MB * 2;
}

# ------------------------------------------------------------------------------
# rif_upload - Request Input Filter for Uploads (record bytes read)
# See also: Apache2::UploadStatus
#
# Server configuration:
#
#   /sys/conf/debug/slow_upload => .3
#
#     Seconds (via sleep) that the input filter waits while reading the request
#     body. This slows down the reading of uploaded data. Used to debug 
#     client-side upload progress indicators.
#
#     Note this doesn't work behind a reverse proxy (such as NGiNX) as it 
#     handles the client upload.
# ------------------------------------------------------------------------------

sub rif_upload {
  my ($class, $f, $bb, $mode, $block, $readbytes) = @_;
  $$Hub{'/sys/log'}->set_request($f->r);
  my $xfrid = $Hub->{'sys'}{'response'}->get('tmp/xfrid');
  if (!$xfrid) {
    $$Hub{'/sys/log'}->debug('TRANSFER: File-upload was not initialized, progress reporting disabled');
    return Apache2::Const::DECLINED;
  }
  my $ctx = $f->ctx || {
    state => 'uploading',
    mtime => 0,
    size => $f->r->headers_in->get('Content-Length') || 0,
    received => 0,
    slow_upload => $$Hub{'/sys/conf/debug/slow_upload'},
  };
  my $status = $f->next->get_brigade($bb, $mode, $block, $readbytes);
  return $status unless $status == APR::Const::SUCCESS;
  $ctx->{received} += $bb->length;
  # Do not write stats more than once per second
  if (!$ctx->{'mtime'} || (time - $ctx->{'mtime'}) > 1
      || $$ctx{'received'} == $$ctx{'size'}) {
    $ctx->{'mtime'} = time;
    $Hub->{'sys'}{'tmp'}->set("xfr/$xfrid.hf", $ctx)->save();
  }
  $f->ctx($ctx);
  if ($$ctx{'slow_upload'}) {
    $$Hub{'/sys/log'}->warn(sprintf('TRANSFER: [%s] received %d of %d',
      $xfrid, $$ctx{'received'}, $$ctx{'size'}));
    sleep($$ctx{'slow_upload'});
  }
  return Apache2::Const::OK;
}

sub compile {
  my $self = shift;
  $Hub->{'/sys/request'}->assert_same_origin();
#
# TODO - This requires all modules which use hub.js to include the auth-token
#
# $Hub->{'/sys/request'}->assert_auth_token();
  my $cgi = $Hub->get('/sys/request/cgi') or die;
  my $resp = $Hub->get('/sys/response') or die;
  $self->run($cgi, $resp);
  $resp->{'headers'}{'Cache-Control'} = 'no-cache, no-store';
  $resp->format;
}

sub run {
  my $self = shift;
  my $verb = $self->{'verb'};
  my $sub = $Commands{$verb} or throw Error::Logical('No such command');
  $self->{'params'} = shift;
  $self->{'result'} = shift;
  $self->{'result'}->set('/head/verb', $verb);
  try {
{'fraction_non_alphanumeric': 0.25516927970915704, 'fraction_numerical': 0.003976369007043854, 'mean_word_length': 2.8190889370932757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3297671', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2959, 'n_words': 794}