svn commit: r585292 - in /spamassassin/trunk/lib/Mail/SpamAssassin: AsyncLoop.pm Conf.pm Dns.pm Plugin/ASN.pm Plugin/URIDNSBL.pm Author: mmartinec Date: Tue Oct 16 17:04:48 2007 New Revision: 585292 URL: http://svn.apache.org/viewvc?rev=585292&view=rev Log: AsyncLoop: replaces sharp clipping on a lower timeout value with a smooth quadratic transition curve from an initial to a minimal timeout value; updated documentation accordingly. Adjust actual timeout value to timer resolution (ceiling) if Time::HiRes module is not available. Conf+AsyncLoop: to a configuration variable rbl_timeout add two optional parameters: a minimal timeout value, and a zone name if a timeout specification should only apply to the specified zone or its subdomains; update documentation accordingly. Modified:     spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm     spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm     spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm     spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm     spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Modified: spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm?rev=585292&r1=585291&r2=585292&view=diff ============================================================================== --- spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm (original) +++ spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm Tue Oct 16 17:04:48 2007 @@ -45,8 +45,15 @@    # Load Time::HiRes if it's available  BEGIN { -  eval { require Time::HiRes }; -  Time::HiRes->import( qw(time) ) unless $@; +  use vars qw($timer_resolution); +  eval { +    require Time::HiRes or die "Error loading Time::HiRes: $@, $!"; +    Time::HiRes->import( qw(time CLOCK_REALTIME) ); +    $timer_resolution = Time::HiRes::clock_getres(CLOCK_REALTIME()); +    1; +  } or do { +    $timer_resolution = 1;  # Perl's builtin timer ticks at one second +  };  }    ############################################################################# @@ -123,27 +130,33 @@    The code reference will be called with one argument, the C<$ent> object.   +=item zone (optional) + +A zone specification (typically a DNS zone name - e.g. host, domain, or RBL) +which is used as a key to look up per-zone settings (currently both timeouts). +  =item timeout_initial (optional)    An initial value of elapsed time for which we are willing to wait for a  response (time in seconds, floating point value is allowed). When elapsed  time since a query started exceeds the timeout value and there are no other -queries to wait for, the query is aborted. The actual timeout value may -shrink dynamically by some factor from a timeout_initial value, the factor -is derived from a number of queries that have already completed, but the -value can not fall below timeout_min (see next parameter). +queries to wait for, the query is aborted. The actual timeout value ranges +from timeout_initial and gradually approaches timeout_min (see next parameter) +as the number of already completed queries approaches the number of all +queries started.    If a caller does not explicitly provide this parameter or its value is  undefined, a default initial timeout value is settable by a configuration  variable rbl_timeout.   -If value of the timeout_initial parameter is already below timeout_min, -it is pushed up to timeout_min. +If a value of the timeout_initial parameter is below timeout_min, the initial +timeout is set to timeout_min.    =item timeout_min (optional)   -A lower bound (in seconds) below which the timeout value can not be -reduced by a dynamic timeout shrinkage formula.  Defaults to 1 second. +A lower bound (in seconds) to which the actual timeout approaches as the +number of queries completed approaches the number of all queries started. +Defaults to 0.2 * timeout_initial.    =back   @@ -164,11 +177,40 @@    $ent->{status} = 'STARTED';    $ent->{start_time} = $now  if !defined $ent->{start_time};   -  $ent->{timeout_min} = 1  if !defined $ent->{timeout_min};  # limits shrinkage -  my $timeout = $ent->{timeout_initial}; -  $timeout = $self->{main}->{conf}->{rbl_timeout}  if !defined $timeout; -  $timeout = $ent->{timeout_min}  if $timeout < $ent->{timeout_min}; -  $ent->{timeout_initial} = $timeout; +  # are there any applicable per-zone settings? +  my $zone = $ent->{zone}; +  my $settings;  # a ref to a by-zone or to global settings +  my $conf_by_zone = $self->{main}->{conf}->{by_zone}; +  if (defined $zone && $conf_by_zone) { +  # dbg("async: searching for by_zone settings for $zone"); +    $zone =~ s/^\\.//;  $zone =~ s/\\.\\z//;  # strip leading and trailing dot +    for (;;) {  # 2.10.example.com, 10.example.com, example.com, com, '' +      if (exists $conf_by_zone->{$zone}) { +        $settings = $conf_by_zone->{$zone}; +        dbg("async: applying by_zone settings for $zone"); +        last; +      } elsif ($zone eq '') { +        last; +      } else {  # strip one level, careful with address literals +        $zone = ($zone =~ /^( (?: [^.] | \\[ (?: \\\\. | [^\\]\\\\] )* \\] )* ) +                            \\. (.*) \\z/xs) ? $2 : ''; +      } +    } +  } + +  my $t_init = $ent->{timeout_initial};  # application-specified has precedence +  $t_init = $settings->{rbl_timeout}  if $settings && !defined $t_init; +  $t_init = $self->{main}->{conf}->{rbl_timeout}  if !defined $t_init; +  $t_init = 0  if !defined $t_init;      # last-resort default, just in case + +  my $t_end  = $ent->{timeout_min};      # application-specified has precedence +  $t_end = $settings->{rbl_timeout_min}  if $settings && !defined $t_end; +  $t_end = 0.2 * $t_init  if !defined $t_end; +  $t_end = 0  if $t_end < 0;  # just in case + +  $t_init = $t_end  if $t_init < $t_end; +  $ent->{timeout_initial} = $t_init; +  $ent->{timeout_min} = $t_end;      $ent->{display_id} =  # identifies entry in debug logging and similar      join(", ", grep { defined } @@ -265,28 +307,29 @@    if (defined $timeout && $timeout > 0 &&        %$pending && $self->{total_queries_started} > 0)    { -    # shrink 'select' timeout if a caller specified unnecessarily long -    # value, beyond the latest deadline of any outstanding request; +    # shrink a 'select' timeout if a caller specified unnecessarily long +    # value beyond the latest deadline of any outstanding request;      # can save needless wait time (up to 1 second in harvest_dnsbl_queries) -    my $timeout_shrink_factor = -           1 - 0.7 * ( ($self->{total_queries_completed} / -                        $self->{total_queries_started}) ** 2 ); +    my $r = $self->{total_queries_completed} / $self->{total_queries_started}; +    my $r2 = $r * $r;  # 0..1      my $max_deadline;      while (my($key,$ent) = each %$pending) { -      my $dt = $ent->{timeout_initial} * $timeout_shrink_factor; -      # don't shrink below a timeout's lower bound -      $dt = $ent->{timeout_min}  if $dt < $ent->{timeout_min}; +      my $t_init = $ent->{timeout_initial}; +      my $dt = $t_init - ($t_init - $ent->{timeout_min}) * $r2;        my $deadline = $ent->{start_time} + $dt; -      if (!defined $max_deadline || $deadline > $max_deadline) { -        $max_deadline = $deadline; -      } +      $max_deadline = $deadline  if $deadline > $max_deadline;      } -    my $sufficient_timeout = $max_deadline - $now; -    $sufficient_timeout = 0  if $sufficient_timeout < 0; -    if (defined $max_deadline && $timeout > $sufficient_timeout) { -      dbg("async: reducing select timeout from %.1f to %.1f s", -          $timeout, $sufficient_timeout); -      $timeout = $sufficient_timeout; +    if (defined $max_deadline) { +      # adjust to timer resolution, only deals with 1s and with fine resolution +      $max_deadline = 1 + int $max_deadline +        if $timer_resolution == 1 && $max_deadline > int $max_deadline; +      my $sufficient_timeout = $max_deadline - $now; +      $sufficient_timeout = 0  if $sufficient_timeout < 0; +      if ($timeout > $sufficient_timeout) { +        dbg("async: reducing select timeout from %.1f to %.1f s", +            $timeout, $sufficient_timeout); +        $timeout = $sufficient_timeout; +      }      }    }   @@ -334,18 +377,16 @@      }        if (%$pending) {  # still any requests outstanding? are they expired? -      my $timeout_shrink_factor = +      my $r =          !$allow_aborting_of_expired || !$self->{total_queries_started} ? 1.0 -          :  1 - 0.7 * ( ($self->{total_queries_completed} / -                          $self->{total_queries_started}) ** 2 ); -      dbg("async: timeout shrink factor: %.2f", -          $timeout_shrink_factor)  if $timeout_shrink_factor != 1; - +        : $self->{total_queries_completed} / $self->{total_queries_started}; +      my $r2 = $r * $r;  # 0..1        while (my($key,$ent) = each %$pending) {          $typecount{$ent->{type}}++; -        my $dt = $ent->{timeout_initial} * $timeout_shrink_factor; -        # don't shrink below a timeout's lower bound -        $dt = $ent->{timeout_min}  if $dt < $ent->{timeout_min}; +        my $t_init = $ent->{timeout_initial}; +        my $dt = $t_init - ($t_init - $ent->{timeout_min}) * $r2; +        # adjust to timer resolution, only deals with 1s and fine resolution +        $dt = 1 + int $dt  if $timer_resolution == 1 && $dt > int $dt;          $allexpired = 0  if $now <= $ent->{start_time} + $dt;        }        dbg("async: queries completed: %d, started: %d", Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?rev=585292&r1=585291&r2=585292&view=diff ============================================================================== --- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original) +++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Tue Oct 16 17:04:48 2007 @@ -2444,25 +2444,35 @@      }    });   -=item rbl_timeout n		(default: 15) +=item rbl_timeout  t  [t_min]  [zone]		(default: 15 3)   -All DNS queries are made at the beginning of a check and we try to read the -results at the end.  This value specifies the maximum period of time to wait -for an DNS query.  If most of the DNS queries have succeeded for a particular -message, then SpamAssassin will not wait for the full period to avoid wasting -time on unresponsive server(s).  For the default 15 second timeout, here is a -chart of queries remaining versus the effective timeout in seconds: - -  queries left    100%  90%  80%  70%  60%  50%  40%  30%  20%  10%  0% -  timeout          15   15   14   14   13   11   10    8    5    3   0 - -In addition, whenever the effective timeout is lowered due to additional query -results returning, the remaining queries are always given at least one more -second before timing out, but the wait time will never exceed rbl_timeout. - -For example, if 20 queries are made at the beginning of a message check and 16 -queries have returned (leaving 20%), the remaining 4 queries must finish -within 5 seconds of the beginning of the check or they will be timed out. +All DNS queries are made at the beginning of a check and we try to read +the results at the end.  This value specifies the maximum period of time +(in seconds) to wait for an DNS query.  If most of the DNS queries have +succeeded for a particular message, then SpamAssassin will not wait for +the full period to avoid wasting time on unresponsive server(s), but will +shrink the timeout according to a percentage of queries already completed. +As the number of queries remaining approaches 0, the timeout value will +gradually approach a t_min value, which is an optional second parameter +and defaults to 0.2 * t.  If t is smaller than t_min, the initial timeout +is set to t_min.  Here is a chart of queries remaining versus the timeout +in seconds, for the default 15 second / 3 second timeout setting: + +  queries left  100%  90%  80%  70%  60%  50%  40%  30%  20%  10%   0% +  timeout        15   14.9 14.5 13.9 13.1 12.0 10.7  9.1  7.3  5.3  3 + +For example, if 20 queries are made at the beginning of a message check +and 16 queries have returned (leaving 20%), the remaining 4 queries should +finish within 7.3 seconds since their query started or they will be timed out. +Note that timed out queries are only aborted when there is nothing else left +for SpamAssassin to do - long evaluation of other rules may grant queries +additional time. + +If a parameter 'zone' is specified (it must end with a letter, which +distinguishes it from other numeric parametrs), then the setting only +applies to DNS queries against the specified DNS domain (host, domain or +RBL (sub)zone).  Matching is case-insensitive, the actual domain may be a +subdomain of the specified zone.    =cut   @@ -2470,7 +2480,29 @@      setting => 'rbl_timeout',      is_admin => 1,      default => 15, -    type => $CONF_TYPE_NUMERIC +    code => sub { +      my ($self, $key, $value, $line) = @_; +      unless (defined $value && $value !~ /^$/) { +	return $MISSING_REQUIRED_VALUE; +      } +      local ($1,$2,$3); +      unless ($value =~ /^        ( [+-]? \\d+ (?: \\. \\d*)? ) +                          (?: \\s+ ( [+-]? \\d+ (?: \\. \\d*)? ) )? +                          (?: \\s+ (\\S* [a-zA-Z]) )? $/xs) { +	return $INVALID_VALUE; +      } +      my $zone = $3; +      if (!defined $zone) {  # a global setting +        $self->{rbl_timeout}     = $1+0; +        $self->{rbl_timeout_min} = $2+0  if defined $2; +      } +      else {  # per-zone settings +        $zone =~ s/^\\.//;  $zone =~ s/\\.$//;  # strip leading and trailing dot +        $zone = lc $zone; +        $self->{by_zone}{$zone}{rbl_timeout}     = $1+0; +        $self->{by_zone}{$zone}{rbl_timeout_min} = $2+0  if defined $2; +      } +    }    });    =item util_rb_tld tld1 tld2 ... Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm?rev=585292&r1=585291&r2=585292&view=diff ============================================================================== --- spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm (original) +++ spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm Tue Oct 16 17:04:48 2007 @@ -110,7 +110,7 @@        my $ent = {        key => $key, -      timeout_initial => $self->{conf}->{rbl_timeout}, +      zone => $host,  # serves to fetch other per-zone settings        type => "DNSBL-".$type,        sets => [ ],  # filled in below        rules => [ ], # filled in below @@ -159,7 +159,7 @@      my $ent = {      key => $key, -    timeout_initial => $self->{conf}->{rbl_timeout}, +    zone => $host,  # serves to fetch other per-zone settings      type => "DNSBL-".$type,      rules => [ $rule ],      # id is filled in after we send the query below Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm?rev=585292&r1=585291&r2=585292&view=diff ============================================================================== --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm (original) +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm Tue Oct 16 17:04:48 2007 @@ -213,15 +213,16 @@          # do the DNS query, have the callback process the result rather than poll for them later      my $zone_index = $index; +    my $zone = $reversed_ip_quad . '.' . $entry->{zone};      my $key = "asnlookup-${zone_index}-$entry->{zone}"; -    my $id = $scanner->{main}->{resolver}->bgsend("${reversed_ip_quad}.$entry->{zone}", 'TXT', undef, sub { +    my $id = $scanner->{main}->{resolver}->bgsend($zone, 'TXT', undef, sub {        my ($pkt, $id, $timestamp) = @_;        $scanner->{async}->set_response_packet($id, $pkt, $key, $timestamp);        $self->process_dns_result($scanner, $pkt, $zone_index);      });      my $ent = {        key=>$key, id=>$id, type=>'TXT', -    # timeout => ...   # defaults to $scanner->{conf}->{rbl_timeout} +      zone => $zone,  # serves to fetch other per-zone settings      };      $scanner->{async}->start_lookup($ent);      dbg("asn: launched DNS TXT query for %s.%s in background", Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm?rev=585292&r1=585291&r2=585292&view=diff ============================================================================== --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm (original) +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Tue Oct 16 17:04:48 2007 @@ -471,7 +471,7 @@    return if $scanner->{async}->get_lookup($key);      # dig $dom ns -  my $ent = $self->start_lookup($scanner, 'NS', +  my $ent = $self->start_lookup($scanner, $dom, 'NS',                                  $self->res_bgsend($scanner, $dom, 'NS', $key),                                  $key);    $ent->{obj} = $obj; @@ -517,7 +517,7 @@    return if $scanner->{async}->get_lookup($key);      # dig $hname a -  my $ent = $self->start_lookup($scanner, 'A', +  my $ent = $self->start_lookup($scanner, $hname, 'A',                                  $self->res_bgsend($scanner, $hname, 'A', $key),                                  $key);    $ent->{obj} = $obj; @@ -562,7 +562,7 @@    my $item = $lookupstr.".".$dnsbl;      # dig $ip txt -  my $ent = $self->start_lookup($scanner, 'DNSBL', +  my $ent = $self->start_lookup($scanner, $item, 'DNSBL',                                $self->res_bgsend($scanner, $item, $qtype, $key),                                $key);    $ent->{obj} = $obj; @@ -650,11 +650,11 @@  # ---------------------------------------------------------------------------    sub start_lookup { -  my ($self, $scanner, $type, $id, $key) = @_; +  my ($self, $scanner, $zone, $type, $id, $key) = @_;      my $ent = {      key => $key, -    timeout_initial => $scanner->{conf}->{rbl_timeout}, +    zone => $zone,  # serves to fetch other per-zone settings      type => "URI-".$type,      id => $id,      completed_callback => sub {