<commit_before>package Mojo::URL;
use Mojo::Base -base;
use overload
  'bool'   => sub {1},
  '""'     => sub { shift->to_string },
  fallback => 1;

use Mojo::Parameters;
use Mojo::Path;
use Mojo::Util qw(punycode_decode punycode_encode url_escape url_unescape);

has [qw(fragment host port scheme userinfo)];
has base => sub { Mojo::URL->new };

sub new { shift->SUPER::new->parse(@_) }

sub authority {
  my ($self, $authority) = @_;

  # New authority
  if (defined $authority) {

    # Userinfo
    if ($authority =~ s/^([^\@]+)\@//) { $self->userinfo(url_unescape $1) }

    # Port
    if ($authority =~ s/:(\d+)$//) { $self->port($1) }

    # Host
    my $host = url_unescape $authority;
    return $host =~ /[^\x00-\x7f]/ ? $self->ihost($host) : $self->host($host);
  }

  # Format
  my $userinfo = $self->userinfo;
  $authority .= url_escape($userinfo, '^A-Za-z0-9\-._~!$&\'()*+,;=:') . '@'
    if $userinfo;
  $authority .= lc($self->ihost || '');
  if (my $port = $self->port) { $authority .= ":$port" }

  return $authority;
}

sub clone {
  my $self = shift;

  my $clone = Mojo::URL->new;
  $clone->scheme($self->scheme);
  $clone->authority($self->authority);
  $clone->path($self->path->clone);
  $clone->query($self->query->clone);
  $clone->fragment($self->fragment);
  $clone->base($self->base->clone) if $self->{base};

  return $clone;
}

sub ihost {
  my $self = shift;

  # Decode
  return $self->host(join '.',
    map { /^xn--(.+)$/ ? punycode_decode($_) : $_ } split /\./, shift)
    if @_;

  # Check if host needs to be encoded
  return undef unless my $host = $self->host;
  return $host unless $host =~ /[^\x00-\x7f]/;

  # Encode
  return join '.',
    map { /[^\x00-\x7f]/ ? ('xn--' . punycode_encode $_) : $_ } split /\./,
    $host;
}

sub is_abs { !!shift->scheme }

sub parse {
  my ($self, $url) = @_;
  return $self unless $url;

  # Official regex
  $url =~ m!(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?!;
  $self->scheme($1);
  $self->authority($2);
  $self->path->parse($3);
  $self->query($4);
  $self->fragment($5);

  return $self;
}

sub path {
  my ($self, $path) = @_;

  # Old path
  $self->{path} ||= Mojo::Path->new;
  return $self->{path} unless $path;

  # New path
  $self->{path} = ref $path ? $path : $self->{path}->merge($path);

  return $self;
}

sub protocol { lc(shift->scheme // '') }

sub query {
  my $self = shift;

  # Old parameters
  my $q = $self->{query} ||= Mojo::Parameters->new;
  return $q unless @_;

  # Replace with list
  if (@_ > 1) { $q->params([])->parse(@_) }

  # Merge with array
  elsif (ref $_[0] eq 'ARRAY') {
    while (my $name = shift @{$_[0]}) {
      my $value = shift @{$_[0]};
      defined $value ? $q->param($name => $value) : $q->remove($name);
    }
  }

  # Append hash
  elsif (ref $_[0] eq 'HASH') { $q->append(%{$_[0]}) }

  # Replace with string
  else { $q->parse($_[0]) }

  return $self;
}

sub to_abs {
  my $self = shift;
  my $base = shift || $self->base->clone;

  # Scheme
  my $abs = $self->clone;
  return $abs if $abs->is_abs;
  $abs->scheme($base->scheme);

  # Authority
  return $abs if $abs->authority;
  $abs->authority($base->authority);

  # Absolute path
  my $path = $abs->path;
  return $abs if $path->leading_slash;

  # Inherit path
  my $base_path = $base->path;
  if (!@{$path->parts}) {
    $path
      = $abs->path($base_path->clone)->path->trailing_slash(0)->canonicalize;

    # Query
    return $abs if length $abs->query->to_string;
    $abs->query($base->query->clone);
  }

  # Merge paths
  else { $abs->path($base_path->clone->merge($path)->canonicalize) }

  return $abs;
}

sub to_rel {
  my $self = shift;
  my $base = shift || $self->base->clone;

  # Scheme and authority
  my $rel = $self->clone->base($base)->scheme(undef);
  $rel->userinfo(undef)->host(undef)->port(undef) if $base->authority;

  # Path
  my @parts      = @{$rel->path->parts};
  my $base_path  = $base->path;
  my @base_parts = @{$base_path->parts};
  pop @base_parts unless $base_path->trailing_slash;
  while (@parts && @base_parts && $parts[0] eq $base_parts[0]) {
    shift @parts;
    shift @base_parts;
  }
  my $path = $rel->path(Mojo::Path->new)->path;
  $path->leading_slash(1) if $rel->authority;
  $path->parts([('..') x @base_parts, @parts]);
  $path->trailing_slash(1) if $self->path->trailing_slash;

  return $rel;
}

sub to_string {
  my $self = shift;

  # Scheme
  my $url = '';
  if (my $scheme = $self->scheme) { $url .= lc "$scheme://" }

  # Authority
  my $authority = $self->authority;
  $url .= $url ? $authority : $authority ? "//$authority" : '';

  # Relative path
  my $path = $self->path;
  if (!$url) { $url .= "$path" }
<commit_msg>use protocol method consistently<commit_after>package Mojo::URL;
use Mojo::Base -base;
use overload
  'bool'   => sub {1},
  '""'     => sub { shift->to_string },
  fallback => 1;

use Mojo::Parameters;
use Mojo::Path;
use Mojo::Util qw(punycode_decode punycode_encode url_escape url_unescape);

has [qw(fragment host port scheme userinfo)];
has base => sub { Mojo::URL->new };

sub new { shift->SUPER::new->parse(@_) }

sub authority {
  my ($self, $authority) = @_;

  # New authority
  if (defined $authority) {

    # Userinfo
    if ($authority =~ s/^([^\@]+)\@//) { $self->userinfo(url_unescape $1) }

    # Port
    if ($authority =~ s/:(\d+)$//) { $self->port($1) }

    # Host
    my $host = url_unescape $authority;
    return $host =~ /[^\x00-\x7f]/ ? $self->ihost($host) : $self->host($host);
  }

  # Format
  my $userinfo = $self->userinfo;
  $authority .= url_escape($userinfo, '^A-Za-z0-9\-._~!$&\'()*+,;=:') . '@'
    if $userinfo;
  $authority .= lc($self->ihost || '');
  if (my $port = $self->port) { $authority .= ":$port" }

  return $authority;
}

sub clone {
  my $self = shift;

  my $clone = Mojo::URL->new;
  $clone->scheme($self->scheme);
  $clone->authority($self->authority);
  $clone->path($self->path->clone);
  $clone->query($self->query->clone);
  $clone->fragment($self->fragment);
  $clone->base($self->base->clone) if $self->{base};

  return $clone;
}

sub ihost {
  my $self = shift;

  # Decode
  return $self->host(join '.',
    map { /^xn--(.+)$/ ? punycode_decode($_) : $_ } split /\./, shift)
    if @_;

  # Check if host needs to be encoded
  return undef unless my $host = $self->host;
  return $host unless $host =~ /[^\x00-\x7f]/;

  # Encode
  return join '.',
    map { /[^\x00-\x7f]/ ? ('xn--' . punycode_encode $_) : $_ } split /\./,
    $host;
}

sub is_abs { !!shift->scheme }

sub parse {
  my ($self, $url) = @_;
  return $self unless $url;

  # Official regex
  $url =~ m!(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?!;
  $self->scheme($1);
  $self->authority($2);
  $self->path->parse($3);
  $self->query($4);
  $self->fragment($5);

  return $self;
}

sub path {
  my ($self, $path) = @_;

  # Old path
  $self->{path} ||= Mojo::Path->new;
  return $self->{path} unless $path;

  # New path
  $self->{path} = ref $path ? $path : $self->{path}->merge($path);

  return $self;
}

sub protocol { lc(shift->scheme // '') }

sub query {
  my $self = shift;

  # Old parameters
  my $q = $self->{query} ||= Mojo::Parameters->new;
  return $q unless @_;

  # Replace with list
  if (@_ > 1) { $q->params([])->parse(@_) }

  # Merge with array
  elsif (ref $_[0] eq 'ARRAY') {
    while (my $name = shift @{$_[0]}) {
      my $value = shift @{$_[0]};
      defined $value ? $q->param($name => $value) : $q->remove($name);
    }
  }

  # Append hash
  elsif (ref $_[0] eq 'HASH') { $q->append(%{$_[0]}) }

  # Replace with string
  else { $q->parse($_[0]) }

  return $self;
}

sub to_abs {
  my $self = shift;
  my $base = shift || $self->base->clone;

  # Scheme
  my $abs = $self->clone;
  return $abs if $abs->is_abs;
  $abs->scheme($base->scheme);

  # Authority
  return $abs if $abs->authority;
  $abs->authority($base->authority);

  # Absolute path
  my $path = $abs->path;
  return $abs if $path->leading_slash;

  # Inherit path
  my $base_path = $base->path;
  if (!@{$path->parts}) {
    $path
      = $abs->path($base_path->clone)->path->trailing_slash(0)->canonicalize;

    # Query
    return $abs if length $abs->query->to_string;
    $abs->query($base->query->clone);
  }

  # Merge paths
  else { $abs->path($base_path->clone->merge($path)->canonicalize) }

  return $abs;
}

sub to_rel {
  my $self = shift;
  my $base = shift || $self->base->clone;

  # Scheme and authority
  my $rel = $self->clone->base($base)->scheme(undef);
  $rel->userinfo(undef)->host(undef)->port(undef) if $base->authority;

  # Path
  my @parts      = @{$rel->path->parts};
  my $base_path  = $base->path;
  my @base_parts = @{$base_path->parts};
  pop @base_parts unless $base_path->trailing_slash;
  while (@parts && @base_parts && $parts[0] eq $base_parts[0]) {
    shift @parts;
    shift @base_parts;
  }
  my $path = $rel->path(Mojo::Path->new)->path;
  $path->leading_slash(1) if $rel->authority;
  $path->parts([('..') x @base_parts, @parts]);
  $path->trailing_slash(1) if $self->path->trailing_slash;

  return $rel;
}

sub to_string {
  my $self = shift;

  # Protocol
  my $url = '';
  if (my $proto = $self->protocol) { $url .= "$proto://" }

  # Authority
  my $authority = $self->authority;
  $url .= $url ? $authority : $authority ? "//$authority" : '';

  # Relative path
  my $path = $self->path;
  if (!$url) { $url .= "$path" }

{'fraction_non_alphanumeric': 0.23284469938389632, 'fraction_numerical': 0.006798385383471425, 'mean_word_length': 3.1825855175477566, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 225, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525609', 'n_tokens_mistral': 3759, 'n_tokens_neox': 3647, 'n_words': 794}