<commit_before># File: AntiSpam.pm
# Author: pragma_
#
# Purpose: Checks if a message is spam

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

package PBot::AntiSpam;

use warnings;
use strict;

use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

use PBot::DualIndexHashObject;

use Carp ();
use Time::HiRes qw(gettimeofday);
use POSIX qw/strftime/;

sub new {
  Carp::croak("Options to " . __FILE__ . " should be key/value pairs, not hash reference") if ref($_[1]) eq 'HASH';
  my ($class, %conf) = @_;
  my $self = bless {}, $class;
  $self->initialize(%conf);
  return $self;
}

sub initialize {
  my ($self, %conf) = @_;
  
  $self->{pbot} = delete $conf{pbot} // Carp::croak("Missing pbot reference to " . __FILE__);

  my $filename = delete $conf{spamkeywords_file} // $self->{pbot}->{registry}->get_value('general', 'data_dir') . '/spam_keywords';
  $self->{keywords} = PBot::DualIndexHashObject->new(name => 'SpamKeywords', filename => $filename);
  $self->{keywords}->load;

  $self->{pbot}->{registry}->add_default('text', 'antispam', 'enforce',  $conf{enforce_antispam} // 1);
  $self->{pbot}->{commands}->register(sub { return $self->antispam_cmd(@_) }, "antispam", 10);
}

sub is_spam {
  my ($self, $namespace, $text, $all_namespaces) = @_;

  return 0 if not $self->{pbot}->{registry}->get_value('antispam', 'enforce');
  return 0 if $self->{pbot}->{registry}->get_value($namespace, 'dont_enforce_antispam');

  my $ret = eval {
    foreach my $space (keys %{ $self->{keywords}->hash }) {
      if ($all_namespaces or $namespace =~ m/^$space$/i) {
        foreach my $keyword (keys %{ $self->{keywords}->hash->{$space} }) {
          return 1 if $text =~ m/$keyword/i;
        }
      }
    }
    return 0;
  };

  if ($@) {
    $self->{pbot}->{logger}->log("Error in is_spam: $@");
    return 0;
  }
  $self->{pbot}->{logger}->log("AntiSpam: spam detected!\n") if $ret;
  return $ret;
}

sub antispam_cmd {
  my ($self, $from, $nick, $user, $host, $arguments) = @_;
  $arguments = lc $arguments;

  my ($command, $args) = split /\s+/, $arguments, 2;

  return "Usage: antispam <command>, where commands are: list/show, add, remove, set, unset" if not defined $command;

  given ($command) {
    when ($_ eq "list" or $_ eq "show") {
      my $text = "Spam keywords:\n";
      my $entries = 0;
      foreach my $namespace (keys %{ $self->{keywords}->hash }) {
        $text .= "  $namespace:\n";
        foreach my $keyword (keys %{ $self->{keywords}->hash->{$namespace} }) {
          $text .= "    $keyword,\n";
          $entries++;
        }
      }
      $text .= "none" if $entries == 0;
      return $text;
    }
    when ("set") {
      my ($namespace, $keyword, $flag, $value) = split /\s+/, $args, 4;
      return "Usage: keywords set <namespace> <keyword> [flag] [value]" if not defined $namespace or not defined $keyword;

      if (not exists $self->{keywords}->hash->{$namespace}) {
        return "There is no such namespace `$namespace` in the keywords.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "There is no such keyword `$keyword` for namespace `$namespace` in the keywords.";
      }

      if (not defined $flag) {
        my $text = "Flags:\n";
        my $comma = '';
        foreach $flag (keys %{ $self->{keywords}->hash->{$namespace}->{$keyword} }) {
          if ($flag eq 'created_on') {
            my $timestamp = strftime "%a %b %e %H:%M:%S %Z %Y", localtime $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
            $text .= $comma . "created_on: $timestamp";
          } else {
            $value = $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
            $text .= $comma .  "$flag: $value";
          }
          $comma = ",\n  ";
        }
        return $text;
      }

      if (not defined $value) {
        $value = $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
        if (not defined $value) {
          return "/say $flag is not set.";
        } else {
          return "/say $flag is set to $value";
        }
      }

      $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag} = $value;
      $self->{keywords}->save;
      return "Flag set.";
    }
    when ("unset") {
      my ($namespace, $keyword, $flag) = split /\s+/, $args, 3;
      return "Usage: keywords unset <namespace> <keyword> <flag>" if not defined $namespace or not defined $keyword or not defined $flag;

      if (not exists $self->{keywords}->hash->{$namespace}) {
        return "There is no such namespace `$namespace` in the keywords.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "There is no such keyword `$keyword` for namespace `$namespace` in the keywords.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag}) {
        return "There is no such flag `$flag` for keyword `$keyword` for namespace `$namespace` in the keywords.";
      }

      delete $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
      $self->{keywords}->save;
      return "Flag unset.";
    }
    when ("add") {
      my ($namespace, $keyword) = split /\s+/, $args, 2;
      return "Usage: keywords add <namespace> <keyword>" if not defined $namespace or not defined $keyword;
      $self->{keywords}->hash->{$namespace}->{$keyword}->{owner} = "$nick!$user\@$host";
      $self->{keywords}->hash->{$namespace}->{$keyword}->{created_on} = gettimeofday;
      $self->{keywords}->save;
      return "/say Added.";
    }
    when ("remove") {
      my ($namespace, $keyword) = split /\s+/, $args, 2;
      return "Usage: keywords remove <namespace> <keyword>" if not defined $namespace or not defined $keyword;

      if(not defined $self->{keywords}->hash->{$namespace}) {
        return "No entries for namespace $namespace";
      }

      if(not defined $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "No such entry for namespace $namespace";
      }

      delete $self->{keywords}->hash->{$namespace}->{$keyword};
      delete $self->{keywords}->hash->{$namespace} if keys %{ $self->{keywords}->hash->{$namespace} } == 0;
      $self->{keywords}->save;
      return "/say Removed.";
    }
    default {
      return "Unknown command '$command'; commands are: list/show, add, remove";
    }
  }
}

1;<commit_msg>AntiSpam: use positional arguments; minor text changes<commit_after># File: AntiSpam.pm
# Author: pragma_
#
# Purpose: Checks if a message is spam

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

package PBot::AntiSpam;

use warnings;
use strict;

use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

use PBot::DualIndexHashObject;

use Carp ();
use Time::HiRes qw(gettimeofday);
use POSIX qw/strftime/;

sub new {
  Carp::croak("Options to " . __FILE__ . " should be key/value pairs, not hash reference") if ref($_[1]) eq 'HASH';
  my ($class, %conf) = @_;
  my $self = bless {}, $class;
  $self->initialize(%conf);
  return $self;
}

sub initialize {
  my ($self, %conf) = @_;
  
  $self->{pbot} = delete $conf{pbot} // Carp::croak("Missing pbot reference to " . __FILE__);

  my $filename = delete $conf{spamkeywords_file} // $self->{pbot}->{registry}->get_value('general', 'data_dir') . '/spam_keywords';
  $self->{keywords} = PBot::DualIndexHashObject->new(name => 'SpamKeywords', filename => $filename);
  $self->{keywords}->load;

  $self->{pbot}->{registry}->add_default('text', 'antispam', 'enforce',  $conf{enforce_antispam} // 1);
  $self->{pbot}->{commands}->register(sub { return $self->antispam_cmd(@_) }, "antispam", 10);
}

sub is_spam {
  my ($self, $namespace, $text, $all_namespaces) = @_;

  return 0 if not $self->{pbot}->{registry}->get_value('antispam', 'enforce');
  return 0 if $self->{pbot}->{registry}->get_value($namespace, 'dont_enforce_antispam');

  my $ret = eval {
    foreach my $space (keys %{ $self->{keywords}->hash }) {
      if ($all_namespaces or $namespace =~ m/^$space$/i) {
        foreach my $keyword (keys %{ $self->{keywords}->hash->{$space} }) {
          return 1 if $text =~ m/$keyword/i;
        }
      }
    }
    return 0;
  };

  if ($@) {
    $self->{pbot}->{logger}->log("Error in is_spam: $@");
    return 0;
  }
  $self->{pbot}->{logger}->log("AntiSpam: spam detected!\n") if $ret;
  return $ret;
}

sub antispam_cmd {
  my ($self, $from, $nick, $user, $host, $arguments, $stuff) = @_;

  my @pargs = @{$stuff->{argumentspos}};

  my $command = shift @pargs;

  return "Usage: antispam <command>, where commands are: list/show, add, remove, set, unset" if not defined $command;

  given ($command) {
    when ($_ eq "list" or $_ eq "show") {
      my $text = "Spam keywords:\n";
      my $entries = 0;
      foreach my $namespace (keys %{ $self->{keywords}->hash }) {
        $text .= "  $namespace:\n";
        foreach my $keyword (keys %{ $self->{keywords}->hash->{$namespace} }) {
          $text .= "    $keyword,\n";
          $entries++;
        }
      }
      $text .= "none" if $entries == 0;
      return $text;
    }
    when ("set") {
      my ($namespace, $keyword, $flag, $value) = $self->{pbot}->{interpreter}->split_args(\@pargs, 4);
      return "Usage: antispam set <namespace> <regex> [flag] [value]" if not defined $namespace or not defined $keyword;

      if (not exists $self->{keywords}->hash->{$namespace}) {
        return "There is no such namespace `$namespace`.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "There is no such regex `$keyword` for namespace `$namespace`.";
      }

      if (not defined $flag) {
        my $text = "Flags:\n";
        my $comma = '';
        foreach $flag (keys %{ $self->{keywords}->hash->{$namespace}->{$keyword} }) {
          if ($flag eq 'created_on') {
            my $timestamp = strftime "%a %b %e %H:%M:%S %Z %Y", localtime $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
            $text .= $comma . "created_on: $timestamp";
          } else {
            $value = $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
            $text .= $comma .  "$flag: $value";
          }
          $comma = ",\n  ";
        }
        return $text;
      }

      if (not defined $value) {
        $value = $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
        if (not defined $value) {
          return "/say $flag is not set.";
        } else {
          return "/say $flag is set to $value";
        }
      }

      $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag} = $value;
      $self->{keywords}->save;
      return "Flag set.";
    }
    when ("unset") {
      my ($namespace, $keyword, $flag) = $self->{pbot}->{interpreter}->split_args(\@pargs, 3);
      return "Usage: antispam unset <namespace> <regex> <flag>" if not defined $namespace or not defined $keyword or not defined $flag;

      if (not exists $self->{keywords}->hash->{$namespace}) {
        return "There is no such namespace `$namespace`.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "There is no such keyword `$keyword` for namespace `$namespace`.";
      }

      if (not exists $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag}) {
        return "There is no such flag `$flag` for regex `$keyword` for namespace `$namespace`.";
      }

      delete $self->{keywords}->hash->{$namespace}->{$keyword}->{$flag};
      $self->{keywords}->save;
      return "Flag unset.";
    }
    when ("add") {
      my ($namespace, $keyword) = $self->{pbot}->{interpreter}->split_args(\@pargs, 2);
      return "Usage: antispam add <namespace> <regex>" if not defined $namespace or not defined $keyword;
      $self->{keywords}->hash->{$namespace}->{$keyword}->{owner} = "$nick!$user\@$host";
      $self->{keywords}->hash->{$namespace}->{$keyword}->{created_on} = gettimeofday;
      $self->{keywords}->save;
      return "/say Added `$keyword`.";
    }
    when ("remove") {
      my ($namespace, $keyword) = $self->{pbot}->{interpreter}->split_args(\@pargs, 2);
      return "Usage: antispam remove <namespace> <regex>" if not defined $namespace or not defined $keyword;

      if(not defined $self->{keywords}->hash->{$namespace}) {
        return "No entries for namespace $namespace";
      }

      if(not defined $self->{keywords}->hash->{$namespace}->{$keyword}) {
        return "No such entry for namespace $namespace";
      }

      delete $self->{keywords}->hash->{$namespace}->{$keyword};
      delete $self->{keywords}->hash->{$namespace} if keys %{ $self->{keywords}->hash->{$namespace} } == 0;
      $self->{keywords}->save;
      return "/say Removed `$keyword`.";
    }
    default {
      return "Unknown command '$command'; commands are: list/show, add, remove";
    }
  }
}

1;
{'fraction_non_alphanumeric': 0.1987082066869301, 'fraction_numerical': 0.0038753799392097265, 'mean_word_length': 2.927484333034915, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 286, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434429', 'n_tokens_mistral': 4485, 'n_tokens_neox': 4298, 'n_words': 1128}