<commit_before>package Mojo::Content::MultiPart;
use Mojo::Base 'Mojo::Content';

use Mojo::Util 'b64_encode';

has parts => sub { [] };

sub body_contains {
  my ($self, $chunk) = @_;
  ($_->headers_contain($chunk) or $_->body_contains($chunk)) and return 1
    for @{$self->parts};
  return undef;
}

sub body_size {
  my $self = shift;

  # Check for existing Content-Lenght header
  if (my $len = $self->headers->content_length) { return $len }

  # Calculate length of whole body
  my $len = my $boundary_len = length($self->build_boundary) + 6;
  $len += $_->header_size + $_->body_size + $boundary_len for @{$self->parts};

  return $len;
}

sub build_boundary {
  my $self = shift;

  # Check for existing boundary
  my $boundary;
  return $boundary if defined($boundary = $self->boundary);

  # Generate and check boundary
  my $size = 1;
  do {
    $boundary = b64_encode join('', map chr(rand 256), 1 .. $size++ * 3);
    $boundary =~ s/\W/X/g;
  } while $self->body_contains($boundary);

  # Add boundary to Content-Type header
  my $headers = $self->headers;
  ($headers->content_type // '') =~ m!^(.*multipart/[^;]+)(.*)$!;
  my $before = $1 || 'multipart/mixed';
  my $after  = $2 || '';
  $headers->content_type("$before; boundary=$boundary$after");

  return $boundary;
}

sub clone {
  my $self = shift;
  return undef unless my $clone = $self->SUPER::clone();
  return $clone->parts($self->parts);
}

sub get_body_chunk {
  my ($self, $offset) = @_;

  # Body generator
  return $self->generate_body_chunk($offset) if $self->{dynamic};

  # First boundary
  my $boundary     = $self->build_boundary;
  my $boundary_len = length($boundary) + 6;
  my $len          = $boundary_len - 2;
  return substr "--$boundary\x0d\x0a", $offset if $len > $offset;

  # Skip parts that have already been processed
  my $start = 0;
  ($len, $start) = ($self->{last_len}, $self->{last_part} + 1)
    if $self->{offset} && $offset > $self->{offset};

  # Prepare content part by part
  my $parts = $self->parts;
  for (my $i = $start; $i < @$parts; $i++) {
    my $part = $parts->[$i];

    # Headers
    my $header_len = $part->header_size;
    return $part->get_header_chunk($offset - $len)
      if ($len + $header_len) > $offset;<commit_msg>cache the boundary too for another little performance boost<commit_after>package Mojo::Content::MultiPart;
use Mojo::Base 'Mojo::Content';

use Mojo::Util 'b64_encode';

has parts => sub { [] };

sub body_contains {
  my ($self, $chunk) = @_;
  ($_->headers_contain($chunk) or $_->body_contains($chunk)) and return 1
    for @{$self->parts};
  return undef;
}

sub body_size {
  my $self = shift;

  # Check for existing Content-Lenght header
  if (my $len = $self->headers->content_length) { return $len }

  # Calculate length of whole body
  my $len = my $boundary_len = length($self->build_boundary) + 6;
  $len += $_->header_size + $_->body_size + $boundary_len for @{$self->parts};

  return $len;
}

sub build_boundary {
  my $self = shift;

  # Check for existing boundary
  my $boundary;
  return $boundary if defined($boundary = $self->boundary);

  # Generate and check boundary
  my $size = 1;
  do {
    $boundary = b64_encode join('', map chr(rand 256), 1 .. $size++ * 3);
    $boundary =~ s/\W/X/g;
  } while $self->body_contains($boundary);

  # Add boundary to Content-Type header
  my $headers = $self->headers;
  ($headers->content_type // '') =~ m!^(.*multipart/[^;]+)(.*)$!;
  my $before = $1 || 'multipart/mixed';
  my $after  = $2 || '';
  $headers->content_type("$before; boundary=$boundary$after");

  return $boundary;
}

sub clone {
  my $self = shift;
  return undef unless my $clone = $self->SUPER::clone();
  return $clone->parts($self->parts);
}

sub get_body_chunk {
  my ($self, $offset) = @_;

  # Body generator
  return $self->generate_body_chunk($offset) if $self->{dynamic};

  # First boundary
  my $boundary     = $self->{boundary} //= $self->build_boundary;
  my $boundary_len = length($boundary) + 6;
  my $len          = $boundary_len - 2;
  return substr "--$boundary\x0d\x0a", $offset if $len > $offset;

  # Skip parts that have already been processed
  my $start = 0;
  ($len, $start) = ($self->{last_len}, $self->{last_part} + 1)
    if $self->{offset} && $offset > $self->{offset};

  # Prepare content part by part
  my $parts = $self->parts;
  for (my $i = $start; $i < @$parts; $i++) {
    my $part = $parts->[$i];

    # Headers
    my $header_len = $part->header_size;
    return $part->get_header_chunk($offset - $len)
      if ($len + $header_len) > $offset;
{'fraction_non_alphanumeric': 0.18026315789473685, 'fraction_numerical': 0.008771929824561403, 'mean_word_length': 3.5885311871227366, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633253', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1563, 'n_words': 430}