svn commit: r586111 - /spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm Author: jm Date: Thu Oct 18 13:09:13 2007 New Revision: 586111 URL: http://svn.apache.org/viewvc?rev=586111&view=rev Log: implement the Bayes chain rule algorithm Modified:     spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm Modified: spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm URL: http://svn.apache.org/viewvc/spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm?rev=586111&r1=586110&r2=586111&view=diff ============================================================================== --- spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm (original) +++ spamassassin/branches/bug-5293-pluginized-bayes/lib/Mail/SpamAssassin/Plugin/OSBF.pm Thu Oct 18 13:09:13 2007 @@ -710,7 +710,8 @@      goto skip;    }   -  $score = Mail::SpamAssassin::Bayes::Combine::combine($ns, $nn, \\@sorted); +  $score = bcr_combine($ns, $nn, \\@sorted); +  # $score = Mail::SpamAssassin::Bayes::Combine::combine($ns, $nn, \\@sorted);      # Couldn't come up with a probability?    goto skip unless defined $score; @@ -769,6 +770,47 @@    return $score;  }   +# combine using the Bayes chain rule algorithm described in the +# EDDC paper + +sub bcr_combine { +  my ($ns, $nn, $probary) = @_; + +  my $PCs = $ns / ($ns + $nn);        # P(Ci) where i = "spam" +  my $PCh = $nn / ($ns + $nn);        # P(Ci) where i = "ham" +  if ($ns + $nn == 0) { +    return;         # cannot combine without both ham *and* spam +  } + +  my $Ps = 0.5;                       # accumulators +  my $Ph = 0.5; +  foreach my $p (@$probary) { +    # note: the denominator is +    # +    #   Sum (i=1 .. m) ( P (Fj | Ci) . P (Ci) ) +    # +    # we only have two classes, s and h, with P(f | s) = $p, P(f | h) = 1-$p +    # +    #   => ($p) * $PCs + (1-$p) * $PCh + +    my $invp = 1-$p; +    my $denom = $p * $PCs + $invp * $PCh; +    $Ps *=    $p / $denom; +    $Ph *= $invp / $denom; + +    # avoid underflow; if they've become ~0, reset to a non-0 value +    # so that further multiplications still work +    if ($Ps == 0) { $Ps = 1e-300; } +    if ($Ph == 0) { $Ph = 1e-300; } +  } + +  # pick the lowest value (a tip from +  # http://www.bgl.nu/bogofilter/BcrFisher.html ) +  if ($Ph < $Ps) { $Ps = 1-$Ph; } else { $Ph = 1-$Ps; } + +  return $Ps; +} +  ###########################################################################    # Plugin hook. @@ -1250,7 +1292,7 @@    # 0.99 instead of 0.125, which makes more sense for us    my $K1 = 0.25;    my $K2 = 10; -  my $K3 = 4; +  my $K3 = 1;      my $Sumf = $s + $n; die "assert: Sumf == 0" unless $Sumf;    my $WdotSumf = $weight * $Sumf;