<commit_before>package Sisimai::MIME;
use strict;
use warnings;
use Encode;
use MIME::Base64 ();
use MIME::QuotedPrint ();
use Sisimai::String;

my $ReE = {
    '7bit-encoded' => qr/^content-transfer-encoding:[ ]*7bit$/m,
    'quoted-print' => qr/^content-transfer-encoding:[ ]*quoted-printable$/m,
    'some-iso2022' => qr/^content-type:[ ]*.+;[ ]*charset=["']?(iso-2022-[-a-z0-9]+)['"]?/m,
    'with-charset' => qr/^content[-]type:[ ]*.+[;][ ]*charset=['"]?([-0-9a-z]+)['"]?/,
    'only-charset' => qr/^[\s\t]+charset=['"]?([-0-9a-z]+)['"]?/,
    'html-message' => qr|^content-type:[ ]*text/html;|m,
};

sub patterns {
  # Make MIME-Encoding and Content-Type related headers regurlar expression
  # @return   [Array] Regular expressions related to MIME encoding
  return $ReE;
}

sub is_mimeencoded {
    # Check that the argument is MIME-Encoded string or not
    # @param    [String] argv1  String to be checked
    # @return   [Integer]       0: Not MIME encoded string
    #                           1: MIME encoded string
    my $class = shift;
    my $argv1 = shift || return 0;
    my @piece = ();
    my $mime1 = 0;

    return undef unless ref $argv1;
    return undef unless ref $argv1 eq 'SCALAR';
    $$argv1 =~ y/"//d;

    if( rindex($$argv1, ' ') > -1 ) {
        # Multiple MIME-Encoded strings in a line
        @piece = split(' ', $$argv1);
    } else {
        push @piece, $$argv1;
    }

    while( my $e = shift @piece ) {
        # Check all the string in the array
        next unless $e =~ /[ \t]*=[?][-_0-9A-Za-z]+[?][BbQq][?].+[?]=?[ \t]*/;
        $mime1 = 1;
    }
    return $mime1;
}

sub mimedecode {
    # Decode MIME-Encoded string
    # @param    [Array] argvs   Reference to an array including MIME-Encoded text
    # @return   [String]        MIME-Decoded text
    my $class = shift;
    my $argvs = shift;

    return '' unless ref $argvs;
    return '' unless ref $argvs eq 'ARRAY';

    my $characterset = '';
    my $encodingname = '';
    my $mimeencoded0 = '';
    my @decodedtext0 = ();
    my $decodedtext1 = '';
    my $utf8decoded1 = '';
    my $notmimetext0 = '';
    my $notmimetext1 = '';

    for my $e ( @$argvs ) {
        # Check and decode each element
        $e =~ s/\A[ \t]+//g;
        $e =~ s/[ \t]+\z//g;
        $e =~ y/"//d;

        if( __PACKAGE__->is_mimeencoded(\$e) ) {
            # MIME Encoded string
            if( $e =~ m{\A(.*)=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=?(.*)\z} ) {
                # =?utf-8?B?55m954yr44Gr44KD44KT44GT?=
                $notmimetext0   = $1;
                $characterset ||= lc $2;
                $encodingname ||= uc $3;
                $mimeencoded0   = $4;
                $notmimetext1   = $5;

                push @decodedtext0, $notmimetext0;
                if( $encodingname eq 'Q' ) {
                    # Quoted-Printable
                    push @decodedtext0, MIME::QuotedPrint::decode($mimeencoded0);

                } elsif( $encodingname eq 'B' ) {
                    # Base64
                    push @decodedtext0, MIME::Base64::decode($mimeencoded0);
                }
                push @decodedtext0, $notmimetext1;
            }
        } else {
            push @decodedtext0, $e;
        }
    }
    return '' unless scalar @decodedtext0;

    $decodedtext1 = join('', @decodedtext0);
    if( $characterset && $encodingname ) {
        # utf-8 => utf8
        $characterset = 'utf8' if $characterset eq 'utf-8';

        if( $characterset ne 'utf8' ) {
            # Characterset is not UTF-8
            eval { Encode::from_to($decodedtext1, $characterset, 'utf8') };
            $decodedtext1 = 'FAILED TO CONVERT THE SUBJECT' if $@;
        }
    }

    $utf8decoded1 = Encode::decode_utf8 $decodedtext1;
    return $utf8decoded1;
}

sub qprintd {
    # Decode MIME Quoted-Printable Encoded string
    # @param    [String] argv1   MIME Encoded text
    # @param    [Hash]   heads   Email header
    # @return   [String]         MIME Decoded text
    my $class = shift;
    my $argv1 = shift // return undef;
    my $heads = shift // {};
    my $plain = '';

    return undef unless ref $argv1;
    return undef unless ref $argv1 eq 'SCALAR';

    if( ! exists $heads->{'content-type'} || ! $heads->{'content-type'} ) {
        # There is no Content-Type: field
        $plain = MIME::QuotedPrint::decode($$argv1);
        return \$plain;
    }

    # Quoted-printable encoded part is the part of the text
    my $boundary00 = __PACKAGE__->boundary($heads->{'content-type'}, 0);
    if( ! $boundary00 || lc($$argv1) !~ $ReE->{'quoted-print'} ) {
        # There is no boundary string or no
        # Content-Transfer-Encoding: quoted-printable field.
        $plain = MIME::QuotedPrint::decode($$argv1);
        return \$plain;
    }

    my $boundary01 = Sisimai::MIME->boundary($heads->{'content-type'}, 1);
    my $bodystring = '';
    my $notdecoded = '';
    my $getencoded = '';
    my $lowercased = '';

    my $encodename = undef;
    my $ctencoding = undef;
    my $mimeinside = 0;

    for my $e ( split("\n", $$argv1) ) {
        # This is a multi-part message in MIME format. Your mail reader does not
        # understand MIME message format.
        # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
        # Content-Type: text/plain; charset=iso-8859-15
        # Content-Transfer-Encoding: quoted-printable
        if( $mimeinside ) {
            # Quoted-Printable encoded text block
            if( $e eq $boundary00 ) {
                # The next boundary string has appeared
                # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
                $getencoded  = MIME::QuotedPrint::decode($notdecoded);
                $getencoded  = Sisimai::String->to_utf8(\$getencoded, $encodename);
                $bodystring .= $$getencoded;
                $bodystring .= $e . "\n";

                $notdecoded = '';
                $mimeinside = 0;
                $ctencoding = undef;
                $encodename = undef;
            } else {
                # Inside of Queoted printable encoded text
                $notdecoded .= $e . "\n";
            }
        } else {
            # NOT Quoted-Printable encoded text block
            $lowercased = lc $e;
            if( $e =~ /\A[-]{2}[^\s]+[^-]\z/ ) {
                # Start of the boundary block
                # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
                unless( $e eq $boundary00 ) {
                    # New boundary string has appeared
                    $boundary00 = $e;
                    $boundary01 = $e . '--';
                }
            } elsif( $lowercased =~ $ReE->{'with-charset'} || $lowercased =~ $ReE->{'only-charset'} ) {
                # Content-Type: text/plain; charset=ISO-2022-JP
                $encodename = $1;
                $mimeinside = 1 if $ctencoding;

            } elsif( $lowercased =~ $ReE->{'quoted-print'} ){
                # Content-Transfer-Encoding: quoted-printable
                $ctencoding = $e;
                $mimeinside = 1 if $encodename;<commit_msg>Scope improvement in Sisimai::MIME<commit_after>package Sisimai::MIME;
use strict;
use warnings;
use Encode;
use MIME::Base64 ();
use MIME::QuotedPrint ();
use Sisimai::String;

my $ReE = {
    '7bit-encoded' => qr/^content-transfer-encoding:[ ]*7bit$/m,
    'quoted-print' => qr/^content-transfer-encoding:[ ]*quoted-printable$/m,
    'some-iso2022' => qr/^content-type:[ ]*.+;[ ]*charset=["']?(iso-2022-[-a-z0-9]+)['"]?/m,
    'with-charset' => qr/^content[-]type:[ ]*.+[;][ ]*charset=['"]?([-0-9a-z]+)['"]?/,
    'only-charset' => qr/^[\s\t]+charset=['"]?([-0-9a-z]+)['"]?/,
    'html-message' => qr|^content-type:[ ]*text/html;|m,
};

sub patterns {
  # Make MIME-Encoding and Content-Type related headers regurlar expression
  # @return   [Array] Regular expressions related to MIME encoding
  return $ReE;
}

sub is_mimeencoded {
    # Check that the argument is MIME-Encoded string or not
    # @param    [String] argv1  String to be checked
    # @return   [Integer]       0: Not MIME encoded string
    #                           1: MIME encoded string
    my $class = shift;
    my $argv1 = shift || return 0;
    my @piece = ();
    my $mime1 = 0;

    return undef unless ref $argv1;
    return undef unless ref $argv1 eq 'SCALAR';
    $$argv1 =~ y/"//d;

    if( rindex($$argv1, ' ') > -1 ) {
        # Multiple MIME-Encoded strings in a line
        @piece = split(' ', $$argv1);
    } else {
        push @piece, $$argv1;
    }

    while( my $e = shift @piece ) {
        # Check all the string in the array
        next unless $e =~ /[ \t]*=[?][-_0-9A-Za-z]+[?][BbQq][?].+[?]=?[ \t]*/;
        $mime1 = 1;
    }
    return $mime1;
}

sub mimedecode {
    # Decode MIME-Encoded string
    # @param    [Array] argvs   Reference to an array including MIME-Encoded text
    # @return   [String]        MIME-Decoded text
    my $class = shift;
    my $argvs = shift;

    return '' unless ref $argvs;
    return '' unless ref $argvs eq 'ARRAY';

    my $characterset = '';
    my $encodingname = '';
    my @decodedtext0 = ();
    my $decodedtext1 = '';
    my $utf8decoded1 = '';

    for my $e ( @$argvs ) {
        # Check and decode each element
        $e =~ s/\A[ \t]+//g;
        $e =~ s/[ \t]+\z//g;
        $e =~ y/"//d;

        if( __PACKAGE__->is_mimeencoded(\$e) ) {
            # MIME Encoded string
            if( $e =~ m{\A(.*)=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=?(.*)\z} ) {
                # =?utf-8?B?55m954yr44Gr44KD44KT44GT?=
                $characterset ||= lc $2;
                $encodingname ||= uc $3;

                push @decodedtext0, $1;
                if( $encodingname eq 'Q' ) {
                    # Quoted-Printable
                    push @decodedtext0, MIME::QuotedPrint::decode($4);

                } elsif( $encodingname eq 'B' ) {
                    # Base64
                    push @decodedtext0, MIME::Base64::decode($4);
                }
                push @decodedtext0, $5;
            }
        } else {
            push @decodedtext0, $e;
        }
    }
    return '' unless scalar @decodedtext0;

    $decodedtext1 = join('', @decodedtext0);
    if( $characterset && $encodingname ) {
        # utf-8 => utf8
        $characterset = 'utf8' if $characterset eq 'utf-8';

        if( $characterset ne 'utf8' ) {
            # Characterset is not UTF-8
            eval { Encode::from_to($decodedtext1, $characterset, 'utf8') };
            $decodedtext1 = 'FAILED TO CONVERT THE SUBJECT' if $@;
        }
    }

    $utf8decoded1 = Encode::decode_utf8 $decodedtext1;
    return $utf8decoded1;
}

sub qprintd {
    # Decode MIME Quoted-Printable Encoded string
    # @param    [String] argv1   MIME Encoded text
    # @param    [Hash]   heads   Email header
    # @return   [String]         MIME Decoded text
    my $class = shift;
    my $argv1 = shift // return undef;
    my $heads = shift // {};
    my $plain = '';

    return undef unless ref $argv1;
    return undef unless ref $argv1 eq 'SCALAR';

    if( ! exists $heads->{'content-type'} || ! $heads->{'content-type'} ) {
        # There is no Content-Type: field
        $plain = MIME::QuotedPrint::decode($$argv1);
        return \$plain;
    }

    # Quoted-printable encoded part is the part of the text
    my $boundary00 = __PACKAGE__->boundary($heads->{'content-type'}, 0);
    if( ! $boundary00 || lc($$argv1) !~ $ReE->{'quoted-print'} ) {
        # There is no boundary string or no
        # Content-Transfer-Encoding: quoted-printable field.
        $plain = MIME::QuotedPrint::decode($$argv1);
        return \$plain;
    }

    my $boundary01 = Sisimai::MIME->boundary($heads->{'content-type'}, 1);
    my $bodystring = '';
    my $notdecoded = '';

    my $encodename = undef;
    my $ctencoding = undef;
    my $mimeinside = 0;

    for my $e ( split("\n", $$argv1) ) {
        # This is a multi-part message in MIME format. Your mail reader does not
        # understand MIME message format.
        # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
        # Content-Type: text/plain; charset=iso-8859-15
        # Content-Transfer-Encoding: quoted-printable
        if( $mimeinside ) {
            # Quoted-Printable encoded text block
            if( $e eq $boundary00 ) {
                # The next boundary string has appeared
                # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
                my $getencoded = MIME::QuotedPrint::decode($notdecoded);
                   $getencoded = Sisimai::String->to_utf8(\$getencoded, $encodename);
                $bodystring .= $$getencoded;
                $bodystring .= $e . "\n";

                $notdecoded = '';
                $mimeinside = 0;
                $ctencoding = undef;
                $encodename = undef;
            } else {
                # Inside of Queoted printable encoded text
                $notdecoded .= $e . "\n";
            }
        } else {
            # NOT Quoted-Printable encoded text block
            my $lowercased = lc $e;
            if( $e =~ /\A[-]{2}[^\s]+[^-]\z/ ) {
                # Start of the boundary block
                # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
                unless( $e eq $boundary00 ) {
                    # New boundary string has appeared
                    $boundary00 = $e;
                    $boundary01 = $e . '--';
                }
            } elsif( $lowercased =~ $ReE->{'with-charset'} || $lowercased =~ $ReE->{'only-charset'} ) {
                # Content-Type: text/plain; charset=ISO-2022-JP
                $encodename = $1;
                $mimeinside = 1 if $ctencoding;

            } elsif( $lowercased =~ $ReE->{'quoted-print'} ){
                # Content-Transfer-Encoding: quoted-printable
                $ctencoding = $e;
                $mimeinside = 1 if $encodename;
{'fraction_non_alphanumeric': 0.15600028795623067, 'fraction_numerical': 0.025556115470448493, 'mean_word_length': 1.8929612661391086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614739', 'n_tokens_mistral': 5022, 'n_tokens_neox': 4685, 'n_words': 1076}