<commit_before>module Sisimai
  module MTA
    # Sisimai::MTA::Qmail parses a bounce email which created by qmail. Methods
    # in the module are called from only Sisimai::Message.
    module Qmail
      # Imported from p5-Sisimail/lib/Sisimai/MTA/qmail.pm
      class << self
        require 'sisimai/mta'
        require 'sisimai/rfc5322'

        Re0 = {
          :subject  => %r/\Afailure notice/i,
          :received => %r/\A[(]qmail[ ]+\d+[ ]+invoked[ ]+(?:for[ ]+bounce|from[ ]+network)[)]/,
        }
        #  qmail-remote.c:248|    if (code >= 500) {
        #  qmail-remote.c:249|      out("h"); outhost(); out(" does not like recipient.\n");
        #  qmail-remote.c:265|  if (code >= 500) quit("D"," failed on DATA command");
        #  qmail-remote.c:271|  if (code >= 500) quit("D"," failed after I sent the message");
        #
        # Characters: K,Z,D in qmail-qmqpc.c, qmail-send.c, qmail-rspawn.c
        #  K = success, Z = temporary error, D = permanent error
        #
        Re1 = {
          :begin  => %r/\AHi[.] This is the qmail/,
          :rfc822 => %r/\A--- Below this line is a copy of the message[.]\z/,
          :error  => %r/\ARemote host said:/,
          :sorry  => %r/\A[Ss]orry[,.][ ]/,
          :endof  => %r/\A__END_OF_EMAIL_MESSAGE__\z/,
        }

        ReSMTP = {
          'conn' => %r{
            # Error text regular expressions which defined in qmail-remote.c
            # qmail-remote.c:225|  if (smtpcode() != 220) quit("ZConnected to "," but greeting failed");
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]greeting[ ]failed[.]
          }x,
          'ehlo' => %r{
            # qmail-remote.c:231|  if (smtpcode() != 250) quit("ZConnected to "," but my name was rejected");
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]my[ ]name[ ]was[ ]rejected[.]
          }x,
          'mail' => %r{
            # qmail-remote.c:238|  if (code >= 500) quit("DConnected to "," but sender was rejected");
            # reason = rejected
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]sender[ ]was[ ]rejected[.]
          }x,
          'rcpt' => %r{
            # qmail-remote.c:249|  out("h"); outhost(); out(" does not like recipient.\n");
            # qmail-remote.c:253|  out("s"); outhost(); out(" does not like recipient.\n");
            # reason = userunknown
            (?:Error:)?.+[ ]does[ ]not[ ]like[ ]recipient[.]
          },
          'data' => %r{(?:
             (?:Error:)?.+[ ]failed[ ]on[ ]DATA[ ]command[.]
            # qmail-remote.c:265|  if (code >= 500) quit("D"," failed on DATA command");
            # qmail-remote.c:266|  if (code >= 400) quit("Z"," failed on DATA command");
            |(?:Error:)?.+[ ]failed[ ]after[ ]I[ ]sent[ ]the[ ]message[.]
            # qmail-remote.c:271|  if (code >= 500) quit("D"," failed after I sent the message");
            # qmail-remote.c:272|  if (code >= 400) quit("Z"," failed after I sent the message");
            )
          }x,
        }

        ReHost = %r{(?:
          # qmail-remote.c:261|  if (!flagbother) quit("DGiving up on ","");
           Giving[ ]up[ ]on[ ](.+[0-9a-zA-Z])[.]?\z
          |Connected[ ]to[ ]([-0-9a-zA-Z.]+[0-9a-zA-Z])[ ]
          |remote[ ]host[ ]([-0-9a-zA-Z.]+[0-9a-zA-Z])[ ]said:
          )
        }x

        ReLDAP = {
          # qmail-ldap-1.03-20040101.patch:19817 - 19866
          'suspend'     => %r/Mailaddress is administrative?le?y disabled/,            # 5.2.1
          'userunknown' => %r/[Ss]orry, no mailbox here by that name/,                 # 5.1.1
          'exceedlimit' => %r/The message exeeded the maximum size the user accepts/,  # 5.2.3
          'systemerror' => %r{(?>
             Automatic[ ]homedir[ ]creator[ ]crashed                    # 4.3.0
            |Illegal[ ]value[ ]in[ ]LDAP[ ]attribute                    # 5.3.5
            |LDAP[ ]attribute[ ]is[ ]not[ ]given[ ]but[ ]mandatory      # 5.3.5
            |Timeout[ ]while[ ]performing[ ]search[ ]on[ ]LDAP[ ]server # 4.4.3
            |Too[ ]many[ ]results[ ]returned[ ]but[ ]needs[ ]to[ ]be[ ]unique # 5.3.5
            |Permanent[ ]error[ ]while[ ]executing[ ]qmail[-]forward    # 5.4.4
            |Temporary[ ](?:
               error[ ](?:
                 in[ ]automatic[ ]homedir[ ]creation            # 4.3.0 or 5.3.0
                |while[ ]executing[ ]qmail[-]forward            # 4.4.4
                )
              |failure[ ]in[ ]LDAP[ ]lookup                       # 4.4.3
              )
            |Unable[ ]to[ ](?:
               contact[ ]LDAP[ ]server                            # 4.4.3
              |login[ ]into[ ]LDAP[ ]server,[ ]bad[ ]credentials  # 4.4.3
              )
            )
          }x,
        }

        # userunknown + expired
        ReOnHold  = %r/\A[^ ]+ does not like recipient[.]\s+.+this message has been in the queue too long[.]\z/
        ReCommand = %r/
          # qmail-remote-fallback.patch
          Sorry,[ ]no[ ]SMTP[ ]connection[ ]got[ ]far[ ]enough;[ ]most[ ]progress[ ]was[ ]([A-Z]{4})[ ]
        /x
        ReFailure = {
          # 'onhold' => %r//,
          'userunknown' => %r{(?:
            # qmail-local.c:589|  strerr_die1x(100,"Sorry, no mailbox here by that name. (#5.1.1)");
             no[ ]mailbox[ ]here[ ]by[ ]that[ ]name
            # qmail-remote.c:253|  out("s"); outhost(); out(" does not like recipient.\n");
            |[ ]does[ ]not[ ]like[ ]recipient[.]
            )
          }x,
          'mailboxfull' => %r{
            # error_str.c:192|  X(EDQUOT,"disk quota exceeded")
            disk[ ]quota[ ]exceeded
          }x,
          'mesgtoobig' => %r{
            # qmail-qmtpd.c:233| ... result = "Dsorry, that message size exceeds my databytes limit (#5.3.4)";
            # qmail-smtpd.c:391| ... out("552 sorry, that message size exceeds my databytes limit (#5.3.4)\r\n"); return;
            Message[ ]size[ ]exceeds[ ]fixed[ ]maximum[ ]message[ ]size:
          }x,
          'hostunknown' => %r{
            # qmail-remote.c:68|  Sorry, I couldn't find any host by that name. (#4.1.2)\n"); zerodie();
            # qmail-remote.c:78|  Sorry, I couldn't find any host named ");
            \ASorry[,][ ]I[ ]couldn[']t[ ]find[ ]any[ ]host[ ]
          }x,
          'systemerror' => %r{(?>
             bad[ ]interpreter:[ ]No[ ]such[ ]file[ ]or[ ]directory
            |system[ ]error
            |Unable[ ]to\b
            )
          }x,
          'networkerror' => %r{Sorry(?:
             [,][ ]I[ ]wasn[']t[ ]able[ ]to[ ]establish[ ]an[ ]SMTP[ ]connection
            |[,][ ]I[ ]couldn[']t[ ]find[ ]a[ ]mail[ ]exchanger[ ]or[ ]IP[ ]address
            |[.][ ]Although[ ]I[']m[ ]listed[ ]as[ ]a[ ]best[-]preference[ ]MX[ ]
                or[ ]A[ ]for[ ]that[ ]host
            )
          }x,
          'systemfull' => %r/Requested action not taken: mailbox unavailable [(]not enough free space[)]/,
        }

        ReDelayed = %r{
          # qmail-send.c:922| ... (&dline[c],"I'm not going to try again; this message has been in the queue too long.\n")) nomem();
          this[ ]message[ ]has[ ]been[ ]in[ ]the[ ]queue[ ]too[ ]long[.]\z
        }x
        Indicators = Sisimai::MTA.INDICATORS
        LongFields = Sisimai::RFC5322.LONGFIELDS
        RFC822Head = Sisimai::RFC5322.HEADERFIELDS

        def description; return 'qmail'; end
        def smtpagent;   return 'qmail'; end
        def headerlist;  return []; end
        def pattern;     return Re0; end

        # Parse bounce messages from qmail
        # @param         [Hash] mhead       Message header of a bounce email
        # @options mhead [String] from      From header
        # @options mhead [String] date      Date header
        # @options mhead [String] subject   Subject header
        # @options mhead [Array]  received  Received headers
        # @options mhead [String] others    Other required headers
        # @param         [String] mbody     Message body of a bounce email
        # @return        [Hash, Nil]        Bounce data list and message/rfc822
        #                                   part or nil if it failed to parse or
        #                                   the arguments are missing
        def scan(mhead, mbody)
          return nil unless mhead
          return nil unless mbody

          # Pre process email headers and the body part of the message which generated
          # by qmail, see http://cr.yp.to/qmail.html
          #   e.g.) Received: (qmail 12345 invoked for bounce); 29 Apr 2009 12:34:56 -0000
          #         Subject: failure notice
          match  = 0
          match += 1 if mhead['subject'] =~ Re0[:subject]
          match += 1 if mhead['received'].find { |a| a =~ Re0[:received] }
          return nil unless match > 0

          dscontents = []; dscontents << Sisimai::MTA.DELIVERYSTATUS
          hasdivided = mbody.split("\n")
          havepassed = [''];
          rfc822next = { 'from' => false, 'to' => false, 'subject' => false }
          rfc822part = ''     # (String) message/rfc822-headers part
          previousfn = ''     # (String) Previous field name
          readcursor = 0      # (Integer) Points the current cursor position
          recipients = 0      # (Integer) The number of 'Final-Recipient' header
          v = nil

          hasdivided.each do |e|
            # Save the current line for the next loop
            havepassed << e; p = havepassed[-2]

            if readcursor == 0
              # Beginning of the bounce message or delivery status part
              if e =~ Re1[:begin]
                readcursor |= Indicators[:'deliverystatus']
                next
              end
            end

            if readcursor & Indicators[:'message-rfc822'] == 0
              # Beginning of the original message part
              if e =~ Re1[:rfc822]<commit_msg>Do not use unless with a operator<commit_after>module Sisimai
  module MTA
    # Sisimai::MTA::Qmail parses a bounce email which created by qmail. Methods
    # in the module are called from only Sisimai::Message.
    module Qmail
      # Imported from p5-Sisimail/lib/Sisimai/MTA/qmail.pm
      class << self
        require 'sisimai/mta'
        require 'sisimai/rfc5322'

        Re0 = {
          :subject  => %r/\Afailure notice/i,
          :received => %r/\A[(]qmail[ ]+\d+[ ]+invoked[ ]+(?:for[ ]+bounce|from[ ]+network)[)]/,
        }
        #  qmail-remote.c:248|    if (code >= 500) {
        #  qmail-remote.c:249|      out("h"); outhost(); out(" does not like recipient.\n");
        #  qmail-remote.c:265|  if (code >= 500) quit("D"," failed on DATA command");
        #  qmail-remote.c:271|  if (code >= 500) quit("D"," failed after I sent the message");
        #
        # Characters: K,Z,D in qmail-qmqpc.c, qmail-send.c, qmail-rspawn.c
        #  K = success, Z = temporary error, D = permanent error
        #
        Re1 = {
          :begin  => %r/\AHi[.] This is the qmail/,
          :rfc822 => %r/\A--- Below this line is a copy of the message[.]\z/,
          :error  => %r/\ARemote host said:/,
          :sorry  => %r/\A[Ss]orry[,.][ ]/,
          :endof  => %r/\A__END_OF_EMAIL_MESSAGE__\z/,
        }

        ReSMTP = {
          'conn' => %r{
            # Error text regular expressions which defined in qmail-remote.c
            # qmail-remote.c:225|  if (smtpcode() != 220) quit("ZConnected to "," but greeting failed");
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]greeting[ ]failed[.]
          }x,
          'ehlo' => %r{
            # qmail-remote.c:231|  if (smtpcode() != 250) quit("ZConnected to "," but my name was rejected");
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]my[ ]name[ ]was[ ]rejected[.]
          }x,
          'mail' => %r{
            # qmail-remote.c:238|  if (code >= 500) quit("DConnected to "," but sender was rejected");
            # reason = rejected
            (?:Error:)?Connected[ ]to[ ].+[ ]but[ ]sender[ ]was[ ]rejected[.]
          }x,
          'rcpt' => %r{
            # qmail-remote.c:249|  out("h"); outhost(); out(" does not like recipient.\n");
            # qmail-remote.c:253|  out("s"); outhost(); out(" does not like recipient.\n");
            # reason = userunknown
            (?:Error:)?.+[ ]does[ ]not[ ]like[ ]recipient[.]
          },
          'data' => %r{(?:
             (?:Error:)?.+[ ]failed[ ]on[ ]DATA[ ]command[.]
            # qmail-remote.c:265|  if (code >= 500) quit("D"," failed on DATA command");
            # qmail-remote.c:266|  if (code >= 400) quit("Z"," failed on DATA command");
            |(?:Error:)?.+[ ]failed[ ]after[ ]I[ ]sent[ ]the[ ]message[.]
            # qmail-remote.c:271|  if (code >= 500) quit("D"," failed after I sent the message");
            # qmail-remote.c:272|  if (code >= 400) quit("Z"," failed after I sent the message");
            )
          }x,
        }

        ReHost = %r{(?:
          # qmail-remote.c:261|  if (!flagbother) quit("DGiving up on ","");
           Giving[ ]up[ ]on[ ](.+[0-9a-zA-Z])[.]?\z
          |Connected[ ]to[ ]([-0-9a-zA-Z.]+[0-9a-zA-Z])[ ]
          |remote[ ]host[ ]([-0-9a-zA-Z.]+[0-9a-zA-Z])[ ]said:
          )
        }x

        ReLDAP = {
          # qmail-ldap-1.03-20040101.patch:19817 - 19866
          'suspend'     => %r/Mailaddress is administrative?le?y disabled/,            # 5.2.1
          'userunknown' => %r/[Ss]orry, no mailbox here by that name/,                 # 5.1.1
          'exceedlimit' => %r/The message exeeded the maximum size the user accepts/,  # 5.2.3
          'systemerror' => %r{(?>
             Automatic[ ]homedir[ ]creator[ ]crashed                    # 4.3.0
            |Illegal[ ]value[ ]in[ ]LDAP[ ]attribute                    # 5.3.5
            |LDAP[ ]attribute[ ]is[ ]not[ ]given[ ]but[ ]mandatory      # 5.3.5
            |Timeout[ ]while[ ]performing[ ]search[ ]on[ ]LDAP[ ]server # 4.4.3
            |Too[ ]many[ ]results[ ]returned[ ]but[ ]needs[ ]to[ ]be[ ]unique # 5.3.5
            |Permanent[ ]error[ ]while[ ]executing[ ]qmail[-]forward    # 5.4.4
            |Temporary[ ](?:
               error[ ](?:
                 in[ ]automatic[ ]homedir[ ]creation            # 4.3.0 or 5.3.0
                |while[ ]executing[ ]qmail[-]forward            # 4.4.4
                )
              |failure[ ]in[ ]LDAP[ ]lookup                       # 4.4.3
              )
            |Unable[ ]to[ ](?:
               contact[ ]LDAP[ ]server                            # 4.4.3
              |login[ ]into[ ]LDAP[ ]server,[ ]bad[ ]credentials  # 4.4.3
              )
            )
          }x,
        }

        # userunknown + expired
        ReOnHold  = %r/\A[^ ]+ does not like recipient[.]\s+.+this message has been in the queue too long[.]\z/
        ReCommand = %r/
          # qmail-remote-fallback.patch
          Sorry,[ ]no[ ]SMTP[ ]connection[ ]got[ ]far[ ]enough;[ ]most[ ]progress[ ]was[ ]([A-Z]{4})[ ]
        /x
        ReFailure = {
          # 'onhold' => %r//,
          'userunknown' => %r{(?:
            # qmail-local.c:589|  strerr_die1x(100,"Sorry, no mailbox here by that name. (#5.1.1)");
             no[ ]mailbox[ ]here[ ]by[ ]that[ ]name
            # qmail-remote.c:253|  out("s"); outhost(); out(" does not like recipient.\n");
            |[ ]does[ ]not[ ]like[ ]recipient[.]
            )
          }x,
          'mailboxfull' => %r{
            # error_str.c:192|  X(EDQUOT,"disk quota exceeded")
            disk[ ]quota[ ]exceeded
          }x,
          'mesgtoobig' => %r{
            # qmail-qmtpd.c:233| ... result = "Dsorry, that message size exceeds my databytes limit (#5.3.4)";
            # qmail-smtpd.c:391| ... out("552 sorry, that message size exceeds my databytes limit (#5.3.4)\r\n"); return;
            Message[ ]size[ ]exceeds[ ]fixed[ ]maximum[ ]message[ ]size:
          }x,
          'hostunknown' => %r{
            # qmail-remote.c:68|  Sorry, I couldn't find any host by that name. (#4.1.2)\n"); zerodie();
            # qmail-remote.c:78|  Sorry, I couldn't find any host named ");
            \ASorry[,][ ]I[ ]couldn[']t[ ]find[ ]any[ ]host[ ]
          }x,
          'systemerror' => %r{(?>
             bad[ ]interpreter:[ ]No[ ]such[ ]file[ ]or[ ]directory
            |system[ ]error
            |Unable[ ]to\b
            )
          }x,
          'networkerror' => %r{Sorry(?:
             [,][ ]I[ ]wasn[']t[ ]able[ ]to[ ]establish[ ]an[ ]SMTP[ ]connection
            |[,][ ]I[ ]couldn[']t[ ]find[ ]a[ ]mail[ ]exchanger[ ]or[ ]IP[ ]address
            |[.][ ]Although[ ]I[']m[ ]listed[ ]as[ ]a[ ]best[-]preference[ ]MX[ ]
                or[ ]A[ ]for[ ]that[ ]host
            )
          }x,
          'systemfull' => %r/Requested action not taken: mailbox unavailable [(]not enough free space[)]/,
        }

        ReDelayed = %r{
          # qmail-send.c:922| ... (&dline[c],"I'm not going to try again; this message has been in the queue too long.\n")) nomem();
          this[ ]message[ ]has[ ]been[ ]in[ ]the[ ]queue[ ]too[ ]long[.]\z
        }x
        Indicators = Sisimai::MTA.INDICATORS
        LongFields = Sisimai::RFC5322.LONGFIELDS
        RFC822Head = Sisimai::RFC5322.HEADERFIELDS

        def description; return 'qmail'; end
        def smtpagent;   return 'qmail'; end
        def headerlist;  return []; end
        def pattern;     return Re0; end

        # Parse bounce messages from qmail
        # @param         [Hash] mhead       Message header of a bounce email
        # @options mhead [String] from      From header
        # @options mhead [String] date      Date header
        # @options mhead [String] subject   Subject header
        # @options mhead [Array]  received  Received headers
        # @options mhead [String] others    Other required headers
        # @param         [String] mbody     Message body of a bounce email
        # @return        [Hash, Nil]        Bounce data list and message/rfc822
        #                                   part or nil if it failed to parse or
        #                                   the arguments are missing
        def scan(mhead, mbody)
          return nil unless mhead
          return nil unless mbody

          # Pre process email headers and the body part of the message which generated
          # by qmail, see http://cr.yp.to/qmail.html
          #   e.g.) Received: (qmail 12345 invoked for bounce); 29 Apr 2009 12:34:56 -0000
          #         Subject: failure notice
          match  = 0
          match += 1 if mhead['subject'] =~ Re0[:subject]
          match += 1 if mhead['received'].find { |a| a =~ Re0[:received] }
          return nil if match == 0

          dscontents = []; dscontents << Sisimai::MTA.DELIVERYSTATUS
          hasdivided = mbody.split("\n")
          havepassed = [''];
          rfc822next = { 'from' => false, 'to' => false, 'subject' => false }
          rfc822part = ''     # (String) message/rfc822-headers part
          previousfn = ''     # (String) Previous field name
          readcursor = 0      # (Integer) Points the current cursor position
          recipients = 0      # (Integer) The number of 'Final-Recipient' header
          v = nil

          hasdivided.each do |e|
            # Save the current line for the next loop
            havepassed << e; p = havepassed[-2]

            if readcursor == 0
              # Beginning of the bounce message or delivery status part
              if e =~ Re1[:begin]
                readcursor |= Indicators[:'deliverystatus']
                next
              end
            end

            if readcursor & Indicators[:'message-rfc822'] == 0
              # Beginning of the original message part
              if e =~ Re1[:rfc822]
{'fraction_non_alphanumeric': 0.1607751385992574, 'fraction_numerical': 0.026855195564823763, 'mean_word_length': 1.6930557457882482, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5968794', 'n_tokens_mistral': 6838, 'n_tokens_neox': 6357, 'n_words': 1729}