<commit_before>package SH::Email::ToHash;

use Mojo::Base -base;
use Data::Printer;
use Data::Dumper;
use Mojo::File 'path';
use MIME::Base64;

#use MIME::Charset;
use MIME::QuotedPrint;
use Clone 'clone';
use open OUT => ':encoding(UTF-8)';
use Encode;

has tmpdir => '/tmp';    # A lot of files will be generated.

# has parser => sub {
#     my $self = shift;
#     my $x    = MIME::Parser->new;
#     path($self->tmpdir)->make_path;
#     $x->output_dir($self->tmpdir);
#     $x;
# };

=encoding UTF-8

=head1 NAME

SH::Email::RawToHash - convert raw email mime text to a nice hash

=head1 SYNOPSIS

    use SH::Email::ToHash;
    use Data::Dumper;
    print Dumper SH::Email::RawToHash::message2hash("From: x@y.c\nTo: d@f.b");

=head1 DESCRIPTION

Export one function to convert raw-email-mime to a nice hash.

=cut


=head2 msgtext2hash

Takes message number and return a hashref

return value {head=>{From=>'...', To=>'...'},body=>{Content-Type=>['text/plain',['charset=iso-8859-1']
Content-Transfer-Encoding=> quoted-printable},message =>{...}}

=cut

sub msgtext2hash {
    my $self   = shift;
    my $msg    = shift || return;
    my $return = {};

    #print @$msg;
#    my $email  = $self->parser->parse_data($msg);

    $msg =~ s&\r&&g;
    my ($header, $body) = split /\n\n/, $msg, 2;

#    say $header;
#
    $return->{header} = $self->parameterify($header);

    #remove first and last line
    my @body_cont = split /\n/, $body;
    if ($body_cont[0] =~ /^\-\-/) {
        $body = '';
        for my $i (1 .. ($#body_cont - 1)) {
            $body .= $body_cont[$i] . "\n";
        }
    }

    #warn $body;
    #warn "###############################################################";
    # TODO: Handle multipart
    if (   exists $return->{header}->{'Content-Type'} ) {

        if ( exists $return->{header}->{'Content-Type'}->{a}
        && $return->{header}->{'Content-Type'}->{a}->[0] =~ /^multipart/) {
            $body = $self->multipart($return->{header}->{'Content-Type'}, $body);    # split or extract body part.
        }
        elsif ( $return->{header}->{'Content-Type'}->{a}->[0] &&( !ref $body || !exists $body->{'Content-Type'} || !$body->{'Content-Type'})) {
            $body={content => $body};
            $body->{'Content-Type'} = $return->{header}->{'Content-Type'}->{a}->[0];
        }
    }
    if ( exists $return->{header}->{'Content-Transfer-Encoding'} && $return->{header}->{'Content-Transfer-Encoding'} && (!ref $body || ! exists $body->{'Content-Transfer-Encoding'} || ! $body->{'Content-Transfer-Encoding'})) {
        if (! ref $body) {
            $body={content => $body};
        }
        $body->{'Content-Transfer-Encoding'} = $return->{header}->{'Content-Transfer-Encoding'};
    }
    if (! ref $body) {
        $return->{body} = $self->parameterify($body);
    } else {
        $return->{body} = $body;
    }
    $return = $self->hash_traverse(
        $return,
        sub {
            my ($v, $k) = @_;
            if ($v && !ref $v) {
                if ($v =~ /^\=.*\?\=/ || $k eq 'content') {
                    $v =~ s/\=\?iso\-8859\-1\?Q\?(.+)\?\=/decode_qp($1)/ige;
                    $v =~ s/\=\?iso\-8859\-1\?B\?(.+)\?\=/decode_base64($1)/ige;
                    $v =~ s/\=\?UTF-8\?B\?(.+)\?\=/decode('UTF-8',decode_base64($1))/ige;
                    $v =~ s/\=\?UTF-8\?Q\?(.+)\?\=/decode('UTF-8',decode_qp($1))/ige;
                }
                return ($v, 'next');
            }
            elsif (defined $k && $k eq 'body' && exists $v->{'Content-Type'} && $v->{'Content-Type'}) {
                if (ref $v->{'Content-Type'} eq 'HASH' && $v->{'Content-Type'}->{a}->[0] =~ /^multipart/) {
                    $v->{content} = $self->multipart($v->{'Content-Type'}, $v->{body});
                }
                else {
                    if ($v->{'Content-Transfer-Encoding'}) {
                        if (lc $v->{'Content-Transfer-Encoding'} eq 'quoted-printable') {
                            $v->{content} = decode_qp($v->{content});
                        }
                        elsif (lc $v->{'Content-Transfer-Encoding'} eq 'base64') {
                            $v->{content} = decode_base64($v->{content});
                        }<commit_msg>perltidy<commit_after>package SH::Email::ToHash;

use Mojo::Base -base;
use Data::Printer;
use Data::Dumper;
use Mojo::File 'path';
use MIME::Base64;

#use MIME::Charset;
use MIME::QuotedPrint;
use Clone 'clone';
use open OUT => ':encoding(UTF-8)';
use utf8;
use Encode;
has tmpdir => '/tmp';    # A lot of files will be generated.

# has parser => sub {
#     my $self = shift;
#     my $x    = MIME::Parser->new;
#     path($self->tmpdir)->make_path;
#     $x->output_dir($self->tmpdir);
#     $x;
# };

=encoding UTF-8

=head1 NAME

SH::Email::RawToHash - convert raw email mime text to a nice hash

=head1 SYNOPSIS

    use SH::Email::RawToHash;
    use Data::Dumper;
    print Dumper SH::Email::RawToHash::message2hash("From: <EMAIL>\nTo: <EMAIL>");

=head1 DESCRIPTION

Export one function to convert raw-email-mime to a nice hash.

=cut


=head2 msgtext2hash

Takes message number and return a hashref

return value {head=>{From=>'...', To=>'...'},body=>{Content-Type=>['text/plain',['charset=iso-8859-1']
Content-Transfer-Encoding=> quoted-printable},message =>{...}}

=cut

sub msgtext2hash {
    my $self   = shift;
    my $msg    = shift || return;
    my $return = {};

    #print @$msg;
#    my $email  = $self->parser->parse_data($msg);

    $msg =~ s&\r&&g;
    my ($header, $body) = split /\n\n/, $msg, 2;

#    say $header;
#
    $return->{header} = $self->parameterify($header);

    #remove first and last line
    my @body_cont = split /\n/, $body;
    if ($body_cont[0] =~ /^\-\-/) {
        $body = '';
        for my $i (1 .. ($#body_cont - 1)) {
            $body .= $body_cont[$i] . "\n";
        }
    }

    #warn $body;
    #warn "###############################################################";
    # TODO: Handle multipart
    if (exists $return->{header}->{'Content-Type'}) {

        if (exists $return->{header}->{'Content-Type'}->{a}
            && $return->{header}->{'Content-Type'}->{a}->[0] =~ /^multipart/) {
            $body = $self->multipart($return->{header}->{'Content-Type'}, $body);    # split or extract body part.
        }
        elsif ($return->{header}->{'Content-Type'}->{a}->[0]
            && (!ref $body || !exists $body->{'Content-Type'} || !$body->{'Content-Type'})) {
            $body = {content => $body};
            $body->{'Content-Type'} = $return->{header}->{'Content-Type'}->{a}->[0];
        }
    }
    if (   exists $return->{header}->{'Content-Transfer-Encoding'}
        && $return->{header}->{'Content-Transfer-Encoding'}
        && (!ref $body || !exists $body->{'Content-Transfer-Encoding'} || !$body->{'Content-Transfer-Encoding'})) {
        if (!ref $body) {
            $body = {content => $body};
        }
        $body->{'Content-Transfer-Encoding'} = $return->{header}->{'Content-Transfer-Encoding'};
    }
    if (!ref $body) {
        $return->{body} = $self->parameterify($body);
    }
    else {
        $return->{body} = $body;
    }
    $return = $self->hash_traverse(
        $return,
        sub {
            my ($v, $k) = @_;
            if ($v && !ref $v) {
                if ($v =~ /^\=.*\?\=/ || $k eq 'content') {
                    $v =~ s/\=\?iso\-8859\-1\?Q\?(.+)\?\=/decode_qp($1)/ige;
                    $v =~ s/\=\?iso\-8859\-1\?B\?(.+)\?\=/decode_base64($1)/ige;
                    $v =~ s/\=\?UTF-8\?B\?(.+)\?\=/decode('UTF-8',decode_base64($1))/ige;
                    $v =~ s/\=\?UTF-8\?Q\?(.+)\?\=/decode('UTF-8',decode_qp($1))/ige;
                }
                return ($v, 'next');
            }
            elsif (defined $k && $k eq 'body' && exists $v->{'Content-Type'} && $v->{'Content-Type'}) {
                if (ref $v->{'Content-Type'} eq 'HASH' && $v->{'Content-Type'}->{a}->[0] =~ /^multipart/) {
                    $v->{content} = $self->multipart($v->{'Content-Type'}, $v->{body});
                }
                else {
                    if ($v->{'Content-Transfer-Encoding'}) {
                        if (lc $v->{'Content-Transfer-Encoding'} eq 'quoted-printable') {
                            $v->{content} = decode_qp($v->{content});
                        }
                        elsif (lc $v->{'Content-Transfer-Encoding'} eq 'base64') {
                            $v->{content} = decode_base64($v->{content});
                        }
{'fraction_non_alphanumeric': 0.24346500996366194, 'fraction_numerical': 0.011839174774352363, 'mean_word_length': 2.1002906976744184, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 153, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512891', 'n_tokens_mistral': 3027, 'n_tokens_neox': 2913, 'n_words': 552}