<commit_before>use strict;
use warnings;

use Test::Differences;
use Test::Fatal;
use Test::More 0.88;

use Courriel::Headers;
use Courriel::Helpers;
use Scalar::Util qw( blessed );

my $crlf = $Courriel::Helpers::CRLF;

my $hola = "\x{00A1}Hola, se\x{00F1}or!";

{
    my $h = Courriel::Headers->new();
    is_deeply(
        _headers_as_arrayref($h),
        [],
        'can make an empty headers object'
    );

    $h->add( Subject => 'Foo bar' );

    is_deeply(
        _headers_as_arrayref($h),
        [ Subject => 'Foo bar' ],
        'added Subject header'
    );

    is_deeply(
        [ map { $_->value() } $h->get('subject') ],
        ['Foo bar'],
        'got subject header (name is case-insensitive)'
    );

    $h->add( 'Content-Type' => 'text/plain' );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Subject        => 'Foo bar',
            'Content-Type' => 'text/plain',
        ],
        'added Content-Type header'
    );

    $h->add( 'Subject' => 'Part 2' );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Subject        => 'Foo bar',
            Subject        => 'Part 2',
            'Content-Type' => 'text/plain',
        ],
        'added another subject header and it shows up after first subject'
    );

    is_deeply(
        [ map { $_->value() } $h->get('subject') ],
        [ 'Foo bar', 'Part 2' ],
        'got all subject headers'
    );

    my $string = <<'EOF';
Subject: Foo bar
Subject: Part 2
Content-Type: text/plain
EOF

    $string =~ s/\n/$crlf/g;

    is(
        $h->as_string(),
        $string,
        'got expected header string'
    );

    $h->remove('Subject');

    is_deeply(
        _headers_as_arrayref($h),
        [
            'Content-Type' => 'text/plain',
        ],
        'removed Subject headers'
    );

    $string = <<'EOF';
Content-Type: text/plain
EOF

    $string =~ s/\n/$crlf/g;

    is(
        $h->as_string(),
        $string,
        'got expected header string'
    );
}

{
    my $headers = <<'EOF';
Foo: 1
Bar: 2
Baz: 3
EOF

    $headers =~ s/\n/$crlf/g;

    my $h = Courriel::Headers->parse( text => \$headers, line_sep => $crlf );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Foo => 1,
            Bar => 2,
            Baz => 3,
        ],
        'parsed simple headers'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1=simple; test2="quoted string"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => 'simple',
            test2 => 'quoted string',
        },
        'parsed attributes with simple values correctly'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1='single'; test2="double"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => 'single',
            test2 => 'double',
        },
        'parsed attributes with simple values correctly'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1="has \\"escaped \\vals"; test2="contains ' single \\quote"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => q{has "escaped vals},
            test2 => q{contains ' single quote},
        },
        'parsed attributes with weird values correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0=foo; val*1=bar});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foobar',
        },
        'parsed attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0="foo bar"; val*1=" baz buz"});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foo bar baz buz',
        },
        'parsed quoted attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0=foo; val*1=" bar"});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foo bar',
        },
        'parsed partially quoted attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*=UTF-8'en-gb'Some%20text%20with%20encoding});

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            'Some text with encoding',
            'UTF-8',
            'en-gb',
        ],
        'parsed encoded attribute correctly'
    );
}


{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*=UTF-8''%e4%b8%80%e4%b8%80});

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            "\x{4E00}\x{4E00}",
            'UTF-8',
            undef,
        ],
        'parsed encoded chinese attribute correctly, with no language'
    );
}


{
    my $extended = <<'EOF';
foo/bar;
  val*0*=UTF-8'en-gb'Some%20text%20with%20encoding;
  val*1=" but now it's quoted and then ";
  val*2=simple;
  val*3*=%20then%20hex%20simple;
EOF

    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes($extended);

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            q{Some text with encoding but now it's quoted and then simple then hex simple},<commit_msg>Add another utf-8 decoding test<commit_after>use strict;
use warnings;

use Test::Differences;
use Test::Fatal;
use Test::More 0.88;

use Courriel::Headers;
use Courriel::Helpers;
use Scalar::Util qw( blessed );

binmode $_, ':utf8'
    for map { Test::Builder->new()->$_() }
    qw( output failure_output todo_output );

my $crlf = $Courriel::Helpers::CRLF;

my $hola = "\x{00A1}Hola, se\x{00F1}or!";

{
    my $h = Courriel::Headers->new();
    is_deeply(
        _headers_as_arrayref($h),
        [],
        'can make an empty headers object'
    );

    $h->add( Subject => 'Foo bar' );

    is_deeply(
        _headers_as_arrayref($h),
        [ Subject => 'Foo bar' ],
        'added Subject header'
    );

    is_deeply(
        [ map { $_->value() } $h->get('subject') ],
        ['Foo bar'],
        'got subject header (name is case-insensitive)'
    );

    $h->add( 'Content-Type' => 'text/plain' );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Subject        => 'Foo bar',
            'Content-Type' => 'text/plain',
        ],
        'added Content-Type header'
    );

    $h->add( 'Subject' => 'Part 2' );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Subject        => 'Foo bar',
            Subject        => 'Part 2',
            'Content-Type' => 'text/plain',
        ],
        'added another subject header and it shows up after first subject'
    );

    is_deeply(
        [ map { $_->value() } $h->get('subject') ],
        [ 'Foo bar', 'Part 2' ],
        'got all subject headers'
    );

    my $string = <<'EOF';
Subject: Foo bar
Subject: Part 2
Content-Type: text/plain
EOF

    $string =~ s/\n/$crlf/g;

    is(
        $h->as_string(),
        $string,
        'got expected header string'
    );

    $h->remove('Subject');

    is_deeply(
        _headers_as_arrayref($h),
        [
            'Content-Type' => 'text/plain',
        ],
        'removed Subject headers'
    );

    $string = <<'EOF';
Content-Type: text/plain
EOF

    $string =~ s/\n/$crlf/g;

    is(
        $h->as_string(),
        $string,
        'got expected header string'
    );
}

{
    my $headers = <<'EOF';
Foo: 1
Bar: 2
Baz: 3
EOF

    $headers =~ s/\n/$crlf/g;

    my $h = Courriel::Headers->parse( text => \$headers, line_sep => $crlf );

    is_deeply(
        _headers_as_arrayref($h),
        [
            Foo => 1,
            Bar => 2,
            Baz => 3,
        ],
        'parsed simple headers'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1=simple; test2="quoted string"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => 'simple',
            test2 => 'quoted string',
        },
        'parsed attributes with simple values correctly'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1='single'; test2="double"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => 'single',
            test2 => 'double',
        },
        'parsed attributes with simple values correctly'
    );
}

{
    my ( $val, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; test1="has \\"escaped \\vals"; test2="contains ' single \\quote"}
        );

    is( $val, 'foo/bar', 'got correct value for header with attributes' );
    is_deeply(
        _attributes_as_hashref($attrs),
        {
            test1 => q{has "escaped vals},
            test2 => q{contains ' single quote},
        },
        'parsed attributes with weird values correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0=foo; val*1=bar});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foobar',
        },
        'parsed attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0="foo bar"; val*1=" baz buz"});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foo bar baz buz',
        },
        'parsed quoted attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*0=foo; val*1=" bar"});

    is_deeply(
        _attributes_as_hashref($attrs),
        {
            val => 'foo bar',
        },
        'parsed partially quoted attribute continuation correctly'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*=UTF-8'en-gb'Some%20text%20with%20encoding});

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            'Some text with encoding',
            'UTF-8',
            'en-gb',
        ],
        'parsed encoded attribute correctly'
    );
}


{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{foo/bar; val*=UTF-8''%e4%b8%80%e4%b8%80});

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            "\x{4E00}\x{4E00}",
            'UTF-8',
            undef,
        ],
        'parsed encoded chinese attribute correctly, with no language'
    );
}

{
    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes(
        q{text/plain; name*=utf-8''Iv%C3%A1n%20F.txt});

    my $attr = $attrs->{name};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            "Iv\x{00E1}n F.txt",
            'UTF-8',
            undef,
        ],
        'parsed encoded European attribute correctly, with no language'
    );
}

{
    my $extended = <<'EOF';
foo/bar;
  val*0*=UTF-8'en-gb'Some%20text%20with%20encoding;
  val*1=" but now it's quoted and then ";
  val*2=simple;
  val*3*=%20then%20hex%20simple;
EOF

    my ( undef, $attrs )
        = Courriel::Helpers::parse_header_with_attributes($extended);

    my $attr = $attrs->{val};

    is_deeply(
        [
            $attr->value(),
            $attr->charset(),
            $attr->language(),
        ],
        [
            q{Some text with encoding but now it's quoted and then simple then hex simple},
{'fraction_non_alphanumeric': 0.1545087692068912, 'fraction_numerical': 0.012959801334781933, 'mean_word_length': 1.861234458259325, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3455195', 'n_tokens_mistral': 4372, 'n_tokens_neox': 4200, 'n_words': 885}