<commit_before>require 'spec_helper'

describe "processing incoming emails" do

  before do
    VerifyDmarc.stub(:call).and_return(true)
  end

  let :params do
    create_incoming_email_params(
      subject:       subject,
      message_id:    message_id,

      from:          from,
      envelope_from: envelope_from,
      sender:        sender,

      recipient:     recipient,
      to:            to,
      cc:            cc,

      content_type:  content_type,
      date:          date,

      in_reply_to:   in_reply_to,
      references:    references,

      body_html:     body_html,
      body_plain:    body_plain,
      stripped_html: stripped_html,
      stripped_text: stripped_text,
      attachments:   attachments,

      thread_index:  thread_index,
      thread_topic:  thread_topic,
    )
  end


  def validate! result
    post emails_url, params
    expect(response).to be_ok

    incoming_email = threadable.incoming_emails.latest

    # before being processed
    expect( params['recipient'] ).to include incoming_email.params['recipient']

    expect( incoming_email.params['timestamp']        ).to eq params['timestamp']
    expect( incoming_email.params['token']            ).to eq params['token']
    expect( incoming_email.params['signature']        ).to eq params['signature']
    expect( incoming_email.params['sender']           ).to eq params['sender']
    expect( incoming_email.params['Sender']           ).to eq params['Sender']
    expect( incoming_email.params['subject']          ).to eq params['subject']
    expect( incoming_email.params['Subject']          ).to eq params['Subject']
    expect( incoming_email.params['from']             ).to eq params['from']
    expect( incoming_email.params['From']             ).to eq params['From']
    expect( incoming_email.params['X-Envelope-From']  ).to eq params['X-Envelope-From']
    expect( incoming_email.params['In-Reply-To']      ).to eq params['In-Reply-To']
    expect( incoming_email.params['References']       ).to eq params['References']
    expect( incoming_email.params['Date']             ).to eq params['Date']
    expect( incoming_email.params['Message-Id']       ).to eq params['Message-Id']
    expect( incoming_email.params['To']               ).to eq params['To']
    expect( incoming_email.params['Cc']               ).to eq params['Cc']
    expect( incoming_email.params['Content-Type']     ).to eq params['Content-Type']
    expect( incoming_email.params['message-headers']  ).to eq params['message-headers']
    expect( incoming_email.params['body-plain']       ).to eq params['body-plain']
    expect( incoming_email.params['body-html']        ).to eq params['body-html']
    expect( incoming_email.params['stripped-html']    ).to eq params['stripped-html']
    expect( incoming_email.params['stripped-text']    ).to eq params['stripped-text']
    expect( incoming_email.params['attachment-count'] ).to eq attachments.count.to_s
    attachments.size.times do |n|
      expect( incoming_email.params["attachment-#{n+1}"] ).to be_present
    end

    expect( recipient ).to include incoming_email.recipient

    expect( incoming_email.subject                ).to eq(subject)
    expect( incoming_email.message_id             ).to eq(message_id)
    expect( incoming_email.from                   ).to eq(from)
    expect( incoming_email.envelope_from          ).to eq(envelope_from)
    expect( incoming_email.sender                 ).to eq(sender)
    expect( incoming_email.to                     ).to eq(to)
    expect( incoming_email.cc                     ).to eq(cc)
    expect( incoming_email.content_type           ).to eq(content_type)
    expect( incoming_email.date                   ).to eq(date)
    expect( incoming_email.in_reply_to            ).to eq(in_reply_to)
    expect( incoming_email.references             ).to eq(references)
    expect( incoming_email.message_headers.to_set ).to eq Set[
      ["X-Envelope-From", envelope_from],
      ["Sender",          sender],
      ["In-Reply-To",     in_reply_to],
      ["References",      references],
      ["From",            from],
      ["Date",            date.rfc2822],
      ["Message-Id",      message_id],
      ["Subject",         subject],
      ["To",              to],
      ["Cc",              cc],
      ["Content-Type",    content_type],
      ["Thread-Index",    thread_index],
      ["Thread-Topic",    thread_topic],
    ]
    expect( incoming_email.body_html      ).to eq(body_html)
    expect( incoming_email.body_plain     ).to eq(body_plain)
    expect( incoming_email.stripped_html  ).to eq(stripped_html)
    expect( incoming_email.stripped_plain ).to eq(stripped_text)

    # process incoming email
    drain_background_jobs!
    incoming_email = threadable.incoming_emails.find_by_id(incoming_email.id) # reload

    # after being processed
    case result
    when :bounced
      expect( incoming_email ).to     be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to_not be_dropped
    when :held
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to     be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to_not be_dropped
    when :dropped
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to     be_dropped
    when :delivered, :duplicate
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to     be_delivered
      expect( incoming_email ).to_not be_dropped
    else
      raise "expect result to :bounced, :held, :delivered, :dropped or :duplicate. got #{result.inspect}"
    end

    expect( incoming_email.organization   ).to eq expected_organization
    expect( incoming_email.parent_message ).to eq expected_parent_message
    expect( incoming_email.conversation   ).to eq expected_conversation
    expect( incoming_email.creator        ).to eq expected_creator

    if result == :bounced || result == :held || result == :dropped
      expect( incoming_email.message                    ).to be_nil
      expect( incoming_email.params['attachment-count'] ).to eq attachments.count.to_s
      expect( incoming_email.params['attachment-1']     ).to be_present
      expect( incoming_email.params['attachment-2']     ).to be_present
      expect( incoming_email.params['attachment-3']     ).to be_present

      if result == :held
        if expect_message_held_notices_sent
          # held mail gets sent an auto-response
          expect(sent_emails.size).to eq 2
          held_notice = sent_emails.with_subject("[message held] #{subject}").first
          expect( held_notice.smtp_envelope_from ).to eq "no-reply-auto@#{thread<EMAIL>host}"
          expect( held_notice.smtp_envelope_to   ).to eq [envelope_from.gsub(/[<>]/, '')]
          expect( held_notice.to                 ).to eq [envelope_from.gsub(/[<>]/, '')]
          expect( held_notice.from               ).to eq ["support+message-held@#{threadable.<EMAIL>_host}"]

          expect( held_notice.header['Reply-To'].to_s       ).to eq "Threadable message held <support+message-<EMAIL>@#{thread<EMAIL>}>"
          expect( held_notice.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
          expect( held_notice.header['References'].to_s     ).to eq incoming_email.message_id
          expect( held_notice.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

          held_owner_notice = sent_emails.with_subject("[message held, action required] #{subject}").first

          expect( held_owner_notice.smtp_envelope_from ).to eq "no-reply-auto@#{threadable.email_host}"
          expect( held_owner_notice.smtp_envelope_to   ).to eq ['<EMAIL>']
          expect( held_owner_notice.to                 ).to eq ['<EMAIL>']
          expect( held_owner_notice.from               ).to eq ["support+message-held@#{threadable.email_host}"]

          expect( held_owner_notice.header['Reply-To'].to_s       ).to eq "Threadable message held <support+message-held@#{threadable.email_host}>"
          expect( held_owner_notice.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
          expect( held_owner_notice.header['References'].to_s     ).to eq incoming_email.message_id
          expect( held_owner_notice.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

        else
          expect(sent_emails.size).to eq 1
        end
      elsif result == :bounced
        expect(sent_emails.size).to eq 1
        bounced_dsn = sent_emails.with_subject("Delivery Status Notification (Failure)").first
        expect( bounced_dsn.smtp_envelope_from ).to eq "no-reply-auto@#{threadable.email_host}"
        expect( bounced_dsn.to                 ).to eq [envelope_from.gsub(/[<>]/, '')]
        expect( bounced_dsn.from               ).to eq ["no-reply-auto@#{threadable.email_host}"]

        expect( bounced_dsn.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
        expect( bounced_dsn.header['References'].to_s     ).to eq incoming_email.message_id
        expect( bounced_dsn.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

        delivery_status_part = bounced_dsn.body.parts[1]
        expect(delivery_status_part.body.to_s).to include "Status: #{expected_bounce_status_code}"
      end

      return<commit_msg>fixed test to expect null envelope froms on bounce dsns<commit_after>require 'spec_helper'

describe "processing incoming emails" do

  before do
    VerifyDmarc.stub(:call).and_return(true)
  end

  let :params do
    create_incoming_email_params(
      subject:       subject,
      message_id:    message_id,

      from:          from,
      envelope_from: envelope_from,
      sender:        sender,

      recipient:     recipient,
      to:            to,
      cc:            cc,

      content_type:  content_type,
      date:          date,

      in_reply_to:   in_reply_to,
      references:    references,

      body_html:     body_html,
      body_plain:    body_plain,
      stripped_html: stripped_html,
      stripped_text: stripped_text,
      attachments:   attachments,

      thread_index:  thread_index,
      thread_topic:  thread_topic,
    )
  end


  def validate! result
    post emails_url, params
    expect(response).to be_ok

    incoming_email = threadable.incoming_emails.latest

    # before being processed
    expect( params['recipient'] ).to include incoming_email.params['recipient']

    expect( incoming_email.params['timestamp']        ).to eq params['timestamp']
    expect( incoming_email.params['token']            ).to eq params['token']
    expect( incoming_email.params['signature']        ).to eq params['signature']
    expect( incoming_email.params['sender']           ).to eq params['sender']
    expect( incoming_email.params['Sender']           ).to eq params['Sender']
    expect( incoming_email.params['subject']          ).to eq params['subject']
    expect( incoming_email.params['Subject']          ).to eq params['Subject']
    expect( incoming_email.params['from']             ).to eq params['from']
    expect( incoming_email.params['From']             ).to eq params['From']
    expect( incoming_email.params['X-Envelope-From']  ).to eq params['X-Envelope-From']
    expect( incoming_email.params['In-Reply-To']      ).to eq params['In-Reply-To']
    expect( incoming_email.params['References']       ).to eq params['References']
    expect( incoming_email.params['Date']             ).to eq params['Date']
    expect( incoming_email.params['Message-Id']       ).to eq params['Message-Id']
    expect( incoming_email.params['To']               ).to eq params['To']
    expect( incoming_email.params['Cc']               ).to eq params['Cc']
    expect( incoming_email.params['Content-Type']     ).to eq params['Content-Type']
    expect( incoming_email.params['message-headers']  ).to eq params['message-headers']
    expect( incoming_email.params['body-plain']       ).to eq params['body-plain']
    expect( incoming_email.params['body-html']        ).to eq params['body-html']
    expect( incoming_email.params['stripped-html']    ).to eq params['stripped-html']
    expect( incoming_email.params['stripped-text']    ).to eq params['stripped-text']
    expect( incoming_email.params['attachment-count'] ).to eq attachments.count.to_s
    attachments.size.times do |n|
      expect( incoming_email.params["attachment-#{n+1}"] ).to be_present
    end

    expect( recipient ).to include incoming_email.recipient

    expect( incoming_email.subject                ).to eq(subject)
    expect( incoming_email.message_id             ).to eq(message_id)
    expect( incoming_email.from                   ).to eq(from)
    expect( incoming_email.envelope_from          ).to eq(envelope_from)
    expect( incoming_email.sender                 ).to eq(sender)
    expect( incoming_email.to                     ).to eq(to)
    expect( incoming_email.cc                     ).to eq(cc)
    expect( incoming_email.content_type           ).to eq(content_type)
    expect( incoming_email.date                   ).to eq(date)
    expect( incoming_email.in_reply_to            ).to eq(in_reply_to)
    expect( incoming_email.references             ).to eq(references)
    expect( incoming_email.message_headers.to_set ).to eq Set[
      ["X-Envelope-From", envelope_from],
      ["Sender",          sender],
      ["In-Reply-To",     in_reply_to],
      ["References",      references],
      ["From",            from],
      ["Date",            date.rfc2822],
      ["Message-Id",      message_id],
      ["Subject",         subject],
      ["To",              to],
      ["Cc",              cc],
      ["Content-Type",    content_type],
      ["Thread-Index",    thread_index],
      ["Thread-Topic",    thread_topic],
    ]
    expect( incoming_email.body_html      ).to eq(body_html)
    expect( incoming_email.body_plain     ).to eq(body_plain)
    expect( incoming_email.stripped_html  ).to eq(stripped_html)
    expect( incoming_email.stripped_plain ).to eq(stripped_text)

    # process incoming email
    drain_background_jobs!
    incoming_email = threadable.incoming_emails.find_by_id(incoming_email.id) # reload

    # after being processed
    case result
    when :bounced
      expect( incoming_email ).to     be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to_not be_dropped
    when :held
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to     be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to_not be_dropped
    when :dropped
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to_not be_delivered
      expect( incoming_email ).to     be_dropped
    when :delivered, :duplicate
      expect( incoming_email ).to_not be_bounced
      expect( incoming_email ).to_not be_held
      expect( incoming_email ).to     be_delivered
      expect( incoming_email ).to_not be_dropped
    else
      raise "expect result to :bounced, :held, :delivered, :dropped or :duplicate. got #{result.inspect}"
    end

    expect( incoming_email.organization   ).to eq expected_organization
    expect( incoming_email.parent_message ).to eq expected_parent_message
    expect( incoming_email.conversation   ).to eq expected_conversation
    expect( incoming_email.creator        ).to eq expected_creator

    if result == :bounced || result == :held || result == :dropped
      expect( incoming_email.message                    ).to be_nil
      expect( incoming_email.params['attachment-count'] ).to eq attachments.count.to_s
      expect( incoming_email.params['attachment-1']     ).to be_present
      expect( incoming_email.params['attachment-2']     ).to be_present
      expect( incoming_email.params['attachment-3']     ).to be_present

      if result == :held
        if expect_message_held_notices_sent
          # held mail gets sent an auto-response
          expect(sent_emails.size).to eq 2
          held_notice = sent_emails.with_subject("[message held] #{subject}").first
          expect( held_notice.smtp_envelope_from ).to eq "no-reply-auto@#{thread<EMAIL>}"
          expect( held_notice.smtp_envelope_to   ).to eq [envelope_from.gsub(/[<>]/, '')]
          expect( held_notice.to                 ).to eq [envelope_from.gsub(/[<>]/, '')]
          expect( held_notice.from               ).to eq ["support+message-<EMAIL>@#{thread<EMAIL>}"]

          expect( held_notice.header['Reply-To'].to_s       ).to eq "Threadable message held <support+<EMAIL>@#{thread<EMAIL>}>"
          expect( held_notice.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
          expect( held_notice.header['References'].to_s     ).to eq incoming_email.message_id
          expect( held_notice.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

          held_owner_notice = sent_emails.with_subject("[message held, action required] #{subject}").first

          expect( held_owner_notice.smtp_envelope_from ).to eq "no-reply-auto@#{thread<EMAIL>_host}"
          expect( held_owner_notice.smtp_envelope_to   ).to eq ['<EMAIL>']
          expect( held_owner_notice.to                 ).to eq ['<EMAIL>']
          expect( held_owner_notice.from               ).to eq ["support+message-held@#{threadable.email_host}"]

          expect( held_owner_notice.header['Reply-To'].to_s       ).to eq "Threadable message held <support+message-held@#{threadable.email_host}>"
          expect( held_owner_notice.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
          expect( held_owner_notice.header['References'].to_s     ).to eq incoming_email.message_id
          expect( held_owner_notice.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

        else
          expect(sent_emails.size).to eq 1
        end
      elsif result == :bounced
        expect(sent_emails.size).to eq 1
        bounced_dsn = sent_emails.with_subject("Delivery Status Notification (Failure)").first
        expect( bounced_dsn.smtp_envelope_from ).to eq ""
        expect( bounced_dsn.to                 ).to eq [envelope_from.gsub(/[<>]/, '')]
        expect( bounced_dsn.from               ).to eq ["no-reply-auto@#{threadable.email_host}"]

        expect( bounced_dsn.header['In-Reply-To'].to_s    ).to eq incoming_email.message_id
        expect( bounced_dsn.header['References'].to_s     ).to eq incoming_email.message_id
        expect( bounced_dsn.header['Auto-Submitted'].to_s ).to eq 'auto-replied'

        delivery_status_part = bounced_dsn.body.parts[1]
        expect(delivery_status_part.body.to_s).to include "Status: #{expected_bounce_status_code}"
      end

      return
{'fraction_non_alphanumeric': 0.11650689982092068, 'fraction_numerical': 0.0012640893289792478, 'mean_word_length': 2.675377468060395, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1455945', 'n_tokens_mistral': 5680, 'n_tokens_neox': 5582, 'n_words': 1211}