<commit_before>namespace :ses_machine do
  desc 'Check email with IMAP for new bounced and spam messages'
  task :check_email_imap => :environment do

    imap = Net::IMAP.new(SesMachine.email_server, SesMachine.email_port, SesMachine.email_use_ssl)
    imap.login(SesMachine.email_account, SesMachine.email_password)
    t = SesMachine.database['mails']

    SesMachine.email_imap_folders.each do |folder|
      imap.select(folder)
      imap.search(['FROM', 'email-bounces.amazonses.com', 'NOT', 'SEEN']).each do |message_id|
        data = imap.fetch(message_id, ['UID', 'ENVELOPE', 'RFC822'])[0]
        imap.store(message_id, "-FLAGS", [:Seen]) # TODO: remove this line
        mail = Mail.read_from_string(data.attr['RFC822'])
        bounce = {
          :uid => data.attr['UID'],
          :message_id => data.attr['ENVELOPE']['message_id'].split('@').first[1..-1],
          :raw_source => data.attr['RFC822'],
          :date => data.attr['ENVELOPE']['date'].to_time.utc
        }
        message_id = data.attr['ENVELOPE']['in_reply_to'].split('@').first[1..-1]
        email = t.find_one('_id' => BSON::ObjectId(message_id))['address'].first
        if mail.bounced?
          if mail.retryable?
            bounce_type = SesMachine::Bounce::TYPES[:soft_bounce]
            SesMachine::Hooks.soft_bounce_hook(email) if SesMachine::Hooks.respond_to?(:soft_bounce_hook)
          else
            bounce_type = SesMachine::Bounce::TYPES[:hard_bounce]
            SesMachine::Hooks.hard_bounce_hook(email) if SesMachine::Hooks.respond_to?(:hard_bounce_hook)
          end
          bounce.merge!(:details => mail.diagnostic_code.to_s)
        # TODO: Add check for spam
        else
          bounce_type = SesMachine::Bounce::TYPES[:unknown]
          SesMachine::Hooks.unknown_hook(email) if SesMachine::Hooks.respond_to?(:unknown_hook)
        end
        <commit_msg>Mark messages as read in rake task<commit_after>namespace :ses_machine do
  desc 'Check email with IMAP for new bounced and spam messages'
  task :check_email_imap => :environment do

    imap = Net::IMAP.new(SesMachine.email_server, SesMachine.email_port, SesMachine.email_use_ssl)
    imap.login(SesMachine.email_account, SesMachine.email_password)
    t = SesMachine.database['mails']

    SesMachine.email_imap_folders.each do |folder|
      imap.select(folder)
      imap.search(['FROM', 'email-bounces.amazonses.com', 'NOT', 'SEEN']).each do |message_id|
        data = imap.fetch(message_id, ['UID', 'ENVELOPE', 'RFC822'])[0]
        # imap.store(message_id, "-FLAGS", [:Seen]) # Keep message as unread
        mail = Mail.read_from_string(data.attr['RFC822'])
        bounce = {
          :uid => data.attr['UID'],
          :message_id => data.attr['ENVELOPE']['message_id'].split('@').first[1..-1],
          :raw_source => data.attr['RFC822'],
          :date => data.attr['ENVELOPE']['date'].to_time.utc
        }
        message_id = data.attr['ENVELOPE']['in_reply_to'].split('@').first[1..-1]
        email = t.find_one('_id' => BSON::ObjectId(message_id))['address'].first
        if mail.bounced?
          if mail.retryable?
            bounce_type = SesMachine::Bounce::TYPES[:soft_bounce]
            SesMachine::Hooks.soft_bounce_hook(email) if SesMachine::Hooks.respond_to?(:soft_bounce_hook)
          else
            bounce_type = SesMachine::Bounce::TYPES[:hard_bounce]
            SesMachine::Hooks.hard_bounce_hook(email) if SesMachine::Hooks.respond_to?(:hard_bounce_hook)
          end
          bounce.merge!(:details => mail.diagnostic_code.to_s)
        # TODO: Add check for spam
        else
          bounce_type = SesMachine::Bounce::TYPES[:unknown]
          SesMachine::Hooks.unknown_hook(email) if SesMachine::Hooks.respond_to?(:unknown_hook)
        end
        
{'fraction_non_alphanumeric': 0.14538725878932066, 'fraction_numerical': 0.007401533174729051, 'mean_word_length': 3.2232142857142856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5800918', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1284, 'n_words': 194}