<commit_before>module Sisimai::Bite::Email
  # Sisimai::Bite::::Email::Notes parses a bounce email which created by Lotus
  # Notes Server. Methods in the module are called from only Sisimai::Message.
  module Notes
    class << self
      # Imported from p5-Sisimail/lib/Sisimai/Bite/Email/Notes.pm
      require 'sisimai/bite/email'

      Indicators = Sisimai::Bite::Email.INDICATORS
      MarkingsOf = {
        message: %r/\A[-]+[ ]+Failure Reasons[ ]+[-]+\z/,
        rfc822:  %r/^[-]+[ ]+Returned Message[ ]+[-]+$/,
      }.freeze

      ReFailure = {
        userunknown: %r{(?:
           User[ ]not[ ]listed[ ]in[ ]public[ ]Name[ ][&][ ]Address[ ]Book
          |ディレクトリのリストにありません
          )
        }x,
        networkerror: %r/Message has exceeded maximum hop count/,
      }.freeze

      def description; return 'Lotus Notes'; end
      def smtpagent;   return Sisimai::Bite.smtpagent(self); end
      def headerlist;  return []; end

      # Parse bounce messages from Lotus Notes
      # @param         [Hash] mhead       Message headers of a bounce email
      # @options mhead [String] from      From header
      # @options mhead [String] date      Date header
      # @options mhead [String] subject   Subject header
      # @options mhead [Array]  received  Received headers
      # @options mhead [String] others    Other required headers
      # @param         [String] mbody     Message body of a bounce email
      # @return        [Hash, Nil]        Bounce data list and message/rfc822
      #                                   part or nil if it failed to parse or
      #                                   the arguments are missing
      def scan(mhead, mbody)
        return nil unless mhead
        return nil unless mbody
        return nil unless mhead['subject'].start_with?('Undeliverable message')

        dscontents = [Sisimai::Bite.DELIVERYSTATUS]
        hasdivided = mbody.split("\n")
        rfc822list = []     # (Array) Each line in message/rfc822 part string
        blanklines = 0      # (Integer) The number of blank lines
        readcursor = 0      # (Integer) Points the current cursor position
        recipients = 0      # (Integer) The number of 'Final-Recipient' header
        characters = ''     # (String) Character set name of the bounce mail
        removedmsg = 'MULTIBYTE CHARACTERS HAVE BEEN REMOVED'
        encodedmsg = ''
        v = nil

        hasdivided.each do |e|
          if readcursor.zero?
            # Beginning of the bounce message or delivery status part
            if e =~ MarkingsOf[:message]
              readcursor |= Indicators[:deliverystatus]
              next
            end
          end

          if (readcursor & Indicators[:'message-rfc822']).zero?
            # Beginning of the original message part
            if e =~ MarkingsOf[:rfc822]
              readcursor |= Indicators[:'message-rfc822']
              next
            end
          end

          if characters.empty?
            # Get character set name
            if cv = mhead['content-type'].match(/\A.+;[ ]*charset=(.+)\z/)
              # Content-Type: text/plain; charset=ISO-2022-JP
              characters = cv[1].downcase
            end
          end

          if readcursor & Indicators[:'message-rfc822'] > 0
            # After "message/rfc822"
            if e.empty?
              blanklines += 1
              break if blanklines > 1
              next
            end
            rfc822list << e

          else
            # Before "message/rfc822"
            next if (readcursor & Indicators[:deliverystatus]).zero?

            # ------- Failure Reasons  --------
            #
            # User not listed in public Name & Address Book
            # <EMAIL>
            #
            # ------- Returned Message --------
            v = dscontents[-1]
            if e =~ /\A[^ ]+[@][^ ]+/
              # <EMAIL>
              if v['recipient']
                # There are multiple recipient addresses in the message body.
                dscontents << Sisimai::Bite.DELIVERYSTATUS
                v = dscontents[-1]
              end
              v['recipient'] ||= e
              recipients += 1

            else
              next if e =~ /\A\z/
              next if e =~ /\A[-]+/

              if e =~ /[^\x20-\x7e]/
                # Error message is not ISO-8859-1
                if characters.size > 0
                  # Try to convert string<commit_msg>Replace regular expressions with String#start_with?<commit_after>module Sisimai::Bite::Email
  # Sisimai::Bite::::Email::Notes parses a bounce email which created by Lotus
  # Notes Server. Methods in the module are called from only Sisimai::Message.
  module Notes
    class << self
      # Imported from p5-Sisimail/lib/Sisimai/Bite/Email/Notes.pm
      require 'sisimai/bite/email'

      Indicators = Sisimai::Bite::Email.INDICATORS
      StartingOf = {
        message: ['------- Failure Reasons '],
        rfc822:  ['------- Returned Message '],
      }.freeze

      ReFailure = {
        userunknown: %r{(?:
           User[ ]not[ ]listed[ ]in[ ]public[ ]Name[ ][&][ ]Address[ ]Book
          |ディレクトリのリストにありません
          )
        }x,
        networkerror: %r/Message has exceeded maximum hop count/,
      }.freeze

      def description; return 'Lotus Notes'; end
      def smtpagent;   return Sisimai::Bite.smtpagent(self); end
      def headerlist;  return []; end

      # Parse bounce messages from Lotus Notes
      # @param         [Hash] mhead       Message headers of a bounce email
      # @options mhead [String] from      From header
      # @options mhead [String] date      Date header
      # @options mhead [String] subject   Subject header
      # @options mhead [Array]  received  Received headers
      # @options mhead [String] others    Other required headers
      # @param         [String] mbody     Message body of a bounce email
      # @return        [Hash, Nil]        Bounce data list and message/rfc822
      #                                   part or nil if it failed to parse or
      #                                   the arguments are missing
      def scan(mhead, mbody)
        return nil unless mhead
        return nil unless mbody
        return nil unless mhead['subject'].start_with?('Undeliverable message')

        dscontents = [Sisimai::Bite.DELIVERYSTATUS]
        hasdivided = mbody.split("\n")
        rfc822list = []     # (Array) Each line in message/rfc822 part string
        blanklines = 0      # (Integer) The number of blank lines
        readcursor = 0      # (Integer) Points the current cursor position
        recipients = 0      # (Integer) The number of 'Final-Recipient' header
        characters = ''     # (String) Character set name of the bounce mail
        removedmsg = 'MULTIBYTE CHARACTERS HAVE BEEN REMOVED'
        encodedmsg = ''
        v = nil

        hasdivided.each do |e|
          if readcursor.zero?
            # Beginning of the bounce message or delivery status part
            if e.start_with?(StartingOf[:message][0])
              readcursor |= Indicators[:deliverystatus]
              next
            end
          end

          if (readcursor & Indicators[:'message-rfc822']).zero?
            # Beginning of the original message part
            if e.start_with?(StartingOf[:rfc822][0])
              readcursor |= Indicators[:'message-rfc822']
              next
            end
          end

          if characters.empty?
            # Get character set name
            if cv = mhead['content-type'].match(/\A.+;[ ]*charset=(.+)\z/)
              # Content-Type: text/plain; charset=ISO-2022-JP
              characters = cv[1].downcase
            end
          end

          if readcursor & Indicators[:'message-rfc822'] > 0
            # After "message/rfc822"
            if e.empty?
              blanklines += 1
              break if blanklines > 1
              next
            end
            rfc822list << e

          else
            # Before "message/rfc822"
            next if (readcursor & Indicators[:deliverystatus]).zero?

            # ------- Failure Reasons  --------
            #
            # User not listed in public Name & Address Book
            # <EMAIL>
            #
            # ------- Returned Message --------
            v = dscontents[-1]
            if e =~ /\A[^ ]+[@][^ ]+/
              # <EMAIL>
              if v['recipient']
                # There are multiple recipient addresses in the message body.
                dscontents << Sisimai::Bite.DELIVERYSTATUS
                v = dscontents[-1]
              end
              v['recipient'] ||= e
              recipients += 1

            else
              next if e.empty?
              next if e.start_with?('-')

              if e =~ /[^\x20-\x7e]/
                # Error message is not ISO-8859-1
                if characters.size > 0
                  # Try to convert string
{'fraction_non_alphanumeric': 0.10277033065236818, 'fraction_numerical': 0.012957998212689902, 'mean_word_length': 1.5405788876276958, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5693696', 'n_tokens_mistral': 2604, 'n_tokens_neox': 2438, 'n_words': 736}