<commit_before>require 'spaminator_plugin/mailer'

class SpaminatorPlugin::Spaminator

  class << self
    def run(environment)
      instance = new(environment)
      instance.run
    end

    def benchmark(environment)
      puts Benchmark.measure { run(environment) }
    end

    def initialize_logger(environment)
      logdir = File.join(RAILS_ROOT, 'log', SpaminatorPlugin.name.underscore)
      File.makedirs(logdir) if !File.exist?(logdir)
      logpath = File.join(logdir, "#{environment.name.to_slug}_#{ENV['RAILS_ENV']}_#{Time.now.strftime('%F_%T')}.log")
      @logger = Logger.new(logpath)
    end

    def log(message)
      @logger << "[#{Time.now.strftime('%F %T %z')}] #{message}\n"
    end
  end


  def initialize(environment)
    @environment = environment
    @settings = Noosfero::Plugin::Settings.new(@environment, SpaminatorPlugin)
    @report = SpaminatorPlugin::Report.new(:environment => environment, 
                                           :total_people => Person.count,
                                           :total_comments => Comment.count)
    self.class.initialize_logger(environment)
  end

  def run
    self.class.log("Starting Spaminator scan")
    start_time = Time.now

    process_all_comments
    process_all_people

    finish(start_time)
  end

  protected

  def finish(start_time)
    finish_report
    @settings.last_run = start_time
    @settings.save!
    self.class.log("Finishing Spaminator scan successfully")
  end

  # TODO considering run everything always
  def on_environment
    [ "profiles.environment_id = ?", @environment.id]
  end

  def comments_to_process
    Comment.joins("JOIN articles ON (comments.source_id = articles.id AND comments.source_type = 'Article') JOIN profiles ON (profiles.id = articles.profile_id)").without_spam.where(on_environment)
  end

  def people_to_process
    Person.visible.non_abusers.where(on_environment)
  end

  def process_all_comments
    self.class.log("Starting to process all comments")
    comments = comments_to_process
    total = comments.count
    pbar = ProgressBar.new("☢ Comments", total)
    comments.each do |comment|
      begin
        process_comment(comment)
      rescue
        register_fail(:comments, comment)
      end
      pbar.inc
    end
    @report.processed_comments = total
    pbar.finish
    self.class.log("All comments processed")
  end

  def process_all_people
    self.class.log("Starting to process all people")
    people = people_to_process
    total = people.count
    pbar = ProgressBar.new("☢ People", total)
    people.find_each do |person|
      process_person_by_comments(person)
      process_person_by_no_network(person)
      pbar.inc
    end
    @report.processed_people = total
    pbar.finish
    self.class.log("All people processed")
  end

  def process_comment(comment)
    self.class.log("Processing Comment[#{comment.id.to_s}]")<commit_msg>spaminator: load progress_bar only on development<commit_after>require 'spaminator_plugin/mailer'

class SpaminatorPlugin::Spaminator

  class << self
    def run(environment)
      instance = new(environment)
      instance.run
    end

    def benchmark(environment)
      puts Benchmark.measure { run(environment) }
    end

    def initialize_logger(environment)
      logdir = File.join(RAILS_ROOT, 'log', SpaminatorPlugin.name.underscore)
      File.makedirs(logdir) if !File.exist?(logdir)
      logpath = File.join(logdir, "#{environment.name.to_slug}_#{ENV['RAILS_ENV']}_#{Time.now.strftime('%F_%T')}.log")
      @logger = Logger.new(logpath)
    end

    def log(message)
      @logger << "[#{Time.now.strftime('%F %T %z')}] #{message}\n"
    end
  end


  def initialize(environment)
    @environment = environment
    @settings = Noosfero::Plugin::Settings.new(@environment, SpaminatorPlugin)
    @report = SpaminatorPlugin::Report.new(:environment => environment, 
                                           :total_people => Person.count,
                                           :total_comments => Comment.count)
    self.class.initialize_logger(environment)
  end

  def run
    self.class.log("Starting Spaminator scan")
    start_time = Time.now

    process_all_comments
    process_all_people

    finish(start_time)
  end

  protected

  def finish(start_time)
    finish_report
    @settings.last_run = start_time
    @settings.save!
    self.class.log("Finishing Spaminator scan successfully")
  end

  # TODO considering run everything always
  def on_environment
    [ "profiles.environment_id = ?", @environment.id]
  end

  def comments_to_process
    Comment.joins("JOIN articles ON (comments.source_id = articles.id AND comments.source_type = 'Article') JOIN profiles ON (profiles.id = articles.profile_id)").without_spam.where(on_environment)
  end

  def people_to_process
    Person.visible.non_abusers.where(on_environment)
  end

  def process_all_comments
    self.class.log("Starting to process all comments")
    comments = comments_to_process
    total = comments.count
    pbar = ProgressBar.new("☢ Comments", total) if Rails.env.development?
    comments.each do |comment|
      begin
        process_comment(comment)
      rescue
        register_fail(:comments, comment)
      end
      pbar.inc if Rails.env.development?
    end
    @report.processed_comments = total
    pbar.finish if Rails.env.development?
    self.class.log("All comments processed")
  end

  def process_all_people
    self.class.log("Starting to process all people")
    people = people_to_process
    total = people.count
    pbar = ProgressBar.new("☢ People", total) if Rails.env.development?
    people.find_each do |person|
      process_person_by_comments(person)
      process_person_by_no_network(person)
      pbar.inc if Rails.env.development?
    end
    @report.processed_people = total
    pbar.finish if Rails.env.development?
    self.class.log("All people processed")
  end

  def process_comment(comment)
    self.class.log("Processing Comment[#{comment.id.to_s}]")
{'fraction_non_alphanumeric': 0.10076615589606928, 'fraction_numerical': 0.0, 'mean_word_length': 3.199300699300699, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7316754', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1794, 'n_words': 370}