<commit_before>package Sisimai::MSP::US::SendGrid;
use parent 'Sisimai::MSP';
use feature ':5.10';
use strict;
use warnings;

my $RxMSP = {
    'from'        => qr/\AMAILER-DAEMON\z/,
    'begin'       => qr/\AThis is an automatically generated message from SendGrid[.]\z/,
    'error'       => qr/\AIf you require assistance with this, please contact SendGrid support[.]\z/,
    'rfc822'      => qr|\AContent-Type: message/rfc822|,
    'endof'       => qr/\A__END_OF_EMAIL_MESSAGE__\z/,
    'return-path' => qr/\A[<]apps[@]sendgrid[.]net[>]\z/,
    'subject'     => qr/\AUndelivered Mail Returned to Sender\z/,
};

sub version     { '4.0.2' }
sub description { 'SendGrid: http://sendgrid.com/' }
sub smtpagent   { 'US::SendGrid' }
sub headerlist  { return [ 'Return-Path' ] }

sub scan {
    # @Description  Detect an error from SendGrid
    # @Param <ref>  (Ref->Hash) Message header
    # @Param <ref>  (Ref->String) Message body
    # @Return       (Ref->Hash) Bounce data list and message/rfc822 part
    my $class = shift;
    my $mhead = shift // return undef;
    my $mbody = shift // return undef;

    return undef unless $mhead->{'subject'}     =~ $RxMSP->{'subject'};
    return undef unless $mhead->{'return-path'};
    return undef unless $mhead->{'return-path'} =~ $RxMSP->{'return-path'};

    my $dscontents = [];    # (Ref->Array) SMTP session errors: message/delivery-status
    my $rfc822head = undef; # (Ref->Array) Required header list in message/rfc822 part
    my $rfc822part = '';    # (String) message/rfc822-headers part
    my $previousfn = '';    # (String) Previous field name

    my $stripedtxt = [ split( "\n", $$mbody ) ];
    my $recipients = 0;     # (Integer) The number of 'Final-Recipient' header
    my $commandtxt = '';    # (String) SMTP Command name begin with the string '>>>'
    my $softbounce = 0;     # (Integer) 1 = Soft bounce
    my $connvalues = 0;     # (Integer) Flag, 1 if all the value of $connheader have been set
    my $connheader = {
        'date'    => '',    # The value of Arrival-Date header
    };

    my $v = undef;
    my $p = undef;
    push @$dscontents, __PACKAGE__->DELIVERYSTATUS;
    $rfc822head = __PACKAGE__->RFC822HEADERS;

    require Sisimai::Time;

    for my $e ( @$stripedtxt ) {
        # Read each line between $RxMSP->{'begin'} and $RxMSP->{'rfc822'}.
        if( ( $e =~ $RxMSP->{'rfc822'} ) .. ( $e =~ $RxMSP->{'endof'} ) ) {
            # After "message/rfc822"
            if( $e =~ m/\A([-0-9A-Za-z]+?)[:][ ]*(.+)\z/ ) {
                # Get required headers only
                my $lhs = $1;
                my $rhs = $2;

                $previousfn = '';
                next unless grep { lc( $lhs ) eq lc( $_ ) } @$rfc822head;

                $previousfn  = $lhs;
                $rfc822part .= $e."\n";

            } elsif( $e =~ m/\A[\s\t]+/ ) {
                # Continued line from the previous line
                $rfc822part .= $e."\n" if $previousfn =~ m/\A(?:From|To|Subject)\z/;
            }

        } else {
            # Before "message/rfc822"
            next unless ( $e =~ $RxMSP->{'begin'} ) .. ( $e =~ $RxMSP->{'rfc822'} );
            next unless length $e;

            if( $connvalues == scalar( keys %$connheader ) ) {
                # Final-Recipient: rfc822; <EMAIL>
                # Original-Recipient: rfc822; <EMAIL>
                # Action: failed
                # Status: 5.1.1
                # Diagnostic-Code: 550 5.1.1 <<EMAIL>>... User Unknown 
                $v = $dscontents->[ -1 ];

                if( $e =~ m/\AFinal-Recipient:[ ]*rfc822;[ ]*([^ ]+)\z/i ) {
                    # Final-Recipient: RFC822; <EMAIL>
                    if( length $v->{'recipient'} ) {
                        # There are multiple recipient addresses in the message body.
                        push @$dscontents, __PACKAGE__->DELIVERYSTATUS;
                        $v = $dscontents->[ -1 ];
                    }
                    $v->{'recipient'} = $1;
                    $recipients++;

                } elsif( $e =~ m/\AAction:[ ]*(.+)\z/i ) {
                    # Action: failed
                    $v->{'action'} = lc $1;

                } elsif( $e =~ m/\AStatus:[ ]*(\d[.]\d+[.]\d+)/i ) {
                    # Status: 5.1.1
                    # Status:5.2.0
                    # Status: 5.1.0 (permanent failure)
                    $v->{'status'} = $1;

                } else {

                    if( $e =~ m/\ADiagnostic-Code:[ ]*(.+)\z/i ) {
                        # Diagnostic-Code: 550 5.1.1 <<EMAIL>>... User Unknown
                        $v->{'diagnosis'} = $1;

                    } elsif( $p =~ m/\ADiagnostic-Code:[ ]*/i && $e =~ m/\A[\s\t]+(.+)\z/ ) {
                        # Continued line of the value of Diagnostic-Code header
                        $v->{'diagnosis'} .= ' '.$1;
                        $e = 'Diagnostic-Code: '.$e;

                    }
                }

            } else {
                # This is an automatically generated message from SendGrid.
                # 
                # I'm sorry to have to tell you that your message was not able to be
                # delivered to one of its intended recipients.
                #
                # If you require assistance with this, please contact SendGrid support.
                # 
                # shironekochan:000000:<<EMAIL>> : 192.0.2.250 : mx.example.jp:[192.0.2.153] :
                #   550 5.1.1 <<EMAIL>>... User Unknown  in RCPT TO
                # 
                # ------------=_1351676802-30315-116783
                # Content-Type: message/delivery-status
                # Content-Disposition: inline
                # Content-Transfer-Encoding: 7bit
                # Content-Description: Delivery Report
                #
                # X-SendGrid-QueueID: 959479146
                # X-SendGrid-Sender: <bounces+61689-10be-kijitora=<EMAIL>>
                # Arrival-Date: 2012-12-31 23-59-59
                if( $e =~ m{.+ in (?:End of )?([A-Z]{4}).*\z} ) {
                    # in RCPT TO, in MAIL FROM, end of DATA
                    $commandtxt = $1;

                } elsif( $e =~ m/\AArrival-Date:[ ]*(.+)\z/ ) {
                    # Arrival-Date: Wed, 29 Apr 2009 16:03:18 +0900
                    next if length $connheader->{'date'};
                    my $r = $1;

                    if( $e =~ m/\AArrival-Date: (\d{4})[-](\d{2})[-](\d{2}) (\d{2})[-](\d{2})[-](\d{2})\z/ ) {
                        # Arrival-Date: 2011-08-12 01-05-05
                        $r .= 'Thu, '.$3.' ';
                        $r .= Sisimai::Time->monthname(0)->[ int($2) - 1 ];
                        $r .= ' '.$1.' '.join( ':', $4, $5, $6 );
                        $r .= ' '.Sisimai::Time->abbr2tz('CDT');
                    }
                    $connheader->{'date'} = $r;
                    $connvalues++;
                }
            }
        } # End of if: rfc822

    } continue {
        # Save the current line for the next loop
        $p = $e;
        $e = undef;
    }

    return undef unless $recipients;
    require Sisimai::String;
    require Sisimai::RFC3463;
    require Sisimai::RFC5322;

    for my $e ( @$dscontents ) {
        # Set default values if each value is empty.
        $e->{'date'}    ||= $mhead->{'date'};
        $e->{'diagnosis'} = Sisimai::String->sweep( $e->{'diagnosis'} );

        if( $e->{'status'} ) {
            # Check softbounce or not
            $softbounce = 1 if $e->{'status'} =~ m/\A4[.]/;

        } else {
            # Get the value of SMTP status code as a pseudo D.S.N.
            if( $e->{'diagnosis'} =~ m/\b([45])\d\d\s*/ ) {
                # 4xx or 5xx
                $softbounce = 1 if $1 == 4;<commit_msg>Improved code for setting delivery status<commit_after>package Sisimai::MSP::US::SendGrid;
use parent 'Sisimai::MSP';
use feature ':5.10';
use strict;
use warnings;

my $RxMSP = {
    'from'        => qr/\AMAILER-DAEMON\z/,
    'begin'       => qr/\AThis is an automatically generated message from SendGrid[.]\z/,
    'error'       => qr/\AIf you require assistance with this, please contact SendGrid support[.]\z/,
    'rfc822'      => qr|\AContent-Type: message/rfc822|,
    'endof'       => qr/\A__END_OF_EMAIL_MESSAGE__\z/,
    'return-path' => qr/\A[<]apps[@]sendgrid[.]net[>]\z/,
    'subject'     => qr/\AUndelivered Mail Returned to Sender\z/,
};

sub version     { '4.0.3' }
sub description { 'SendGrid: http://sendgrid.com/' }
sub smtpagent   { 'US::SendGrid' }
sub headerlist  { return [ 'Return-Path' ] }

sub scan {
    # @Description  Detect an error from SendGrid
    # @Param <ref>  (Ref->Hash) Message header
    # @Param <ref>  (Ref->String) Message body
    # @Return       (Ref->Hash) Bounce data list and message/rfc822 part
    my $class = shift;
    my $mhead = shift // return undef;
    my $mbody = shift // return undef;

    return undef unless $mhead->{'subject'}     =~ $RxMSP->{'subject'};
    return undef unless $mhead->{'return-path'};
    return undef unless $mhead->{'return-path'} =~ $RxMSP->{'return-path'};

    my $dscontents = [];    # (Ref->Array) SMTP session errors: message/delivery-status
    my $rfc822head = undef; # (Ref->Array) Required header list in message/rfc822 part
    my $rfc822part = '';    # (String) message/rfc822-headers part
    my $previousfn = '';    # (String) Previous field name

    my $stripedtxt = [ split( "\n", $$mbody ) ];
    my $recipients = 0;     # (Integer) The number of 'Final-Recipient' header
    my $commandtxt = '';    # (String) SMTP Command name begin with the string '>>>'
    my $connvalues = 0;     # (Integer) Flag, 1 if all the value of $connheader have been set
    my $connheader = {
        'date'    => '',    # The value of Arrival-Date header
    };

    my $v = undef;
    my $p = undef;
    push @$dscontents, __PACKAGE__->DELIVERYSTATUS;
    $rfc822head = __PACKAGE__->RFC822HEADERS;

    require Sisimai::Time;

    for my $e ( @$stripedtxt ) {
        # Read each line between $RxMSP->{'begin'} and $RxMSP->{'rfc822'}.
        if( ( $e =~ $RxMSP->{'rfc822'} ) .. ( $e =~ $RxMSP->{'endof'} ) ) {
            # After "message/rfc822"
            if( $e =~ m/\A([-0-9A-Za-z]+?)[:][ ]*(.+)\z/ ) {
                # Get required headers only
                my $lhs = $1;
                my $rhs = $2;

                $previousfn = '';
                next unless grep { lc( $lhs ) eq lc( $_ ) } @$rfc822head;

                $previousfn  = $lhs;
                $rfc822part .= $e."\n";

            } elsif( $e =~ m/\A[\s\t]+/ ) {
                # Continued line from the previous line
                $rfc822part .= $e."\n" if $previousfn =~ m/\A(?:From|To|Subject)\z/;
            }

        } else {
            # Before "message/rfc822"
            next unless ( $e =~ $RxMSP->{'begin'} ) .. ( $e =~ $RxMSP->{'rfc822'} );
            next unless length $e;

            if( $connvalues == scalar( keys %$connheader ) ) {
                # Final-Recipient: rfc822; <EMAIL>
                # Original-Recipient: rfc822; <EMAIL>
                # Action: failed
                # Status: 5.1.1
                # Diagnostic-Code: 550 5.1.1 <<EMAIL>>... User Unknown 
                $v = $dscontents->[ -1 ];

                if( $e =~ m/\AFinal-Recipient:[ ]*rfc822;[ ]*([^ ]+)\z/i ) {
                    # Final-Recipient: RFC822; <EMAIL>
                    if( length $v->{'recipient'} ) {
                        # There are multiple recipient addresses in the message body.
                        push @$dscontents, __PACKAGE__->DELIVERYSTATUS;
                        $v = $dscontents->[ -1 ];
                    }
                    $v->{'recipient'} = $1;
                    $recipients++;

                } elsif( $e =~ m/\AAction:[ ]*(.+)\z/i ) {
                    # Action: failed
                    $v->{'action'} = lc $1;

                } elsif( $e =~ m/\AStatus:[ ]*(\d[.]\d+[.]\d+)/i ) {
                    # Status: 5.1.1
                    # Status:5.2.0
                    # Status: 5.1.0 (permanent failure)
                    $v->{'status'} = $1;

                } else {

                    if( $e =~ m/\ADiagnostic-Code:[ ]*(.+)\z/i ) {
                        # Diagnostic-Code: 550 5.1.1 <<EMAIL>>... User Unknown
                        $v->{'diagnosis'} = $1;

                    } elsif( $p =~ m/\ADiagnostic-Code:[ ]*/i && $e =~ m/\A[\s\t]+(.+)\z/ ) {
                        # Continued line of the value of Diagnostic-Code header
                        $v->{'diagnosis'} .= ' '.$1;
                        $e = 'Diagnostic-Code: '.$e;

                    }
                }

            } else {
                # This is an automatically generated message from SendGrid.
                # 
                # I'm sorry to have to tell you that your message was not able to be
                # delivered to one of its intended recipients.
                #
                # If you require assistance with this, please contact SendGrid support.
                # 
                # shironekochan:000000:<<EMAIL>> : 192.0.2.250 : mx.example.jp:[192.0.2.153] :
                #   550 5.1.1 <<EMAIL>>... User Unknown  in RCPT TO
                # 
                # ------------=_1351676802-30315-116783
                # Content-Type: message/delivery-status
                # Content-Disposition: inline
                # Content-Transfer-Encoding: 7bit
                # Content-Description: Delivery Report
                #
                # X-SendGrid-QueueID: 959479146
                # X-SendGrid-Sender: <<EMAIL>+6<EMAIL>-1<EMAIL>>
                # Arrival-Date: 2012-12-31 23-59-59
                if( $e =~ m{.+ in (?:End of )?([A-Z]{4}).*\z} ) {
                    # in RCPT TO, in MAIL FROM, end of DATA
                    $commandtxt = $1;

                } elsif( $e =~ m/\AArrival-Date:[ ]*(.+)\z/ ) {
                    # Arrival-Date: Wed, 29 Apr 2009 16:03:18 +0900
                    next if length $connheader->{'date'};
                    my $r = $1;

                    if( $e =~ m/\AArrival-Date: (\d{4})[-](\d{2})[-](\d{2}) (\d{2})[-](\d{2})[-](\d{2})\z/ ) {
                        # Arrival-Date: 2011-08-12 01-05-05
                        $r .= 'Thu, '.$3.' ';
                        $r .= Sisimai::Time->monthname(0)->[ int($2) - 1 ];
                        $r .= ' '.$1.' '.join( ':', $4, $5, $6 );
                        $r .= ' '.Sisimai::Time->abbr2tz('CDT');
                    }
                    $connheader->{'date'} = $r;
                    $connvalues++;
                }
            }
        } # End of if: rfc822

    } continue {
        # Save the current line for the next loop
        $p = $e;
        $e = undef;
    }

    return undef unless $recipients;
    require Sisimai::String;
    require Sisimai::RFC3463;
    require Sisimai::RFC5322;

    for my $e ( @$dscontents ) {
        # Set default values if each value is empty.
        $e->{'date'}    ||= $mhead->{'date'};
        $e->{'diagnosis'} = Sisimai::String->sweep( $e->{'diagnosis'} );

        if( $e->{'status'} ) {
            # Check softbounce or not
            $e->{'softbounce'} = 1 if $e->{'status'} =~ m/\A4[.]/;

        } else {
            # Get the value of SMTP status code as a pseudo D.S.N.
            if( $e->{'diagnosis'} =~ m/\b([45])\d\d\s*/ ) {
                # 4xx or 5xx
                $e->{'softbounce'} = 1 if $1 == 4;
{'fraction_non_alphanumeric': 0.16965900284164298, 'fraction_numerical': 0.03274347713769052, 'mean_word_length': 1.435897435897436, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5806653', 'n_tokens_mistral': 5544, 'n_tokens_neox': 4968, 'n_words': 1195}