<commit_before>package Mojo::Util;
use Mojo::Base 'Exporter';

require Digest::MD5;
require Digest::SHA;
require Encode;
require MIME::Base64;
require MIME::QuotedPrint;

# Punycode bootstring parameters
use constant {
  PUNYCODE_BASE         => 36,
  PUNYCODE_TMIN         => 1,
  PUNYCODE_TMAX         => 26,
  PUNYCODE_SKEW         => 38,
  PUNYCODE_DAMP         => 700,
  PUNYCODE_INITIAL_BIAS => 72,
  PUNYCODE_INITIAL_N    => 128
};

# Punycode delimiter
my $DELIMITER = chr 0x2D;

# HTML5 entities for html_unescape (without "apos")
my %ENTITIES = (
  Aacute   => 193,
  aacute   => 225,
  Acirc    => 194,
  acirc    => 226,
  acute    => 180,
  AElig    => 198,
  aelig    => 230,
  Agrave   => 192,
  agrave   => 224,
  alefsym  => 8501,
  Alpha    => 913,
  alpha    => 945,
  amp      => 38,
  and      => 8743,
  ang      => 8736,
  '#39'    => 39,
  Aring    => 197,
  aring    => 229,
  asymp    => 8776,
  Atilde   => 195,
  atilde   => 227,
  Auml     => 196,
  auml     => 228,
  bdquo    => 8222,
  Beta     => 914,
  beta     => 946,
  brvbar   => 166,
  bull     => 8226,
  cap      => 8745,
  Ccedil   => 199,
  ccedil   => 231,
  cedil    => 184,
  cent     => 162,
  Chi      => 935,
  chi      => 967,
  circ     => 710,
  clubs    => 9827,
  cong     => 8773,
  copy     => 169,
  crarr    => 8629,
  cup      => 8746,
  curren   => 164,
  Dagger   => 8225,
  dagger   => 8224,
  dArr     => 8659,
  darr     => 8595,
  deg      => 176,
  Delta    => 916,
  delta    => 948,
  diams    => 9830,
  divide   => 247,
  Eacute   => 201,
  eacute   => 233,
  Ecirc    => 202,
  ecirc    => 234,
  Egrave   => 200,
  egrave   => 232,
  empty    => 8709,
  emsp     => 8195,
  ensp     => 8194,
  Epsilon  => 917,
  epsilon  => 949,
  equiv    => 8801,
  Eta      => 919,
  eta      => 951,
  ETH      => 208,
  eth      => 240,
  Euml     => 203,
  euml     => 235,
  euro     => 8364,
  exist    => 8707,
  fnof     => 402,
  forall   => 8704,
  frac12   => 189,
  frac14   => 188,
  frac34   => 190,
  frasl    => 8260,
  Gamma    => 915,
  gamma    => 947,
  ge       => 8805,
  gt       => 62,
  hArr     => 8660,
  harr     => 8596,
  hearts   => 9829,
  hellip   => 8230,
  Iacute   => 205,
  iacute   => 237,
  Icirc    => 206,
  icirc    => 238,
  iexcl    => 161,
  Igrave   => 204,
  igrave   => 236,
  image    => 8465,
  infin    => 8734,
  int      => 8747,
  Iota     => 921,
  iota     => 953,
  iquest   => 191,
  isin     => 8712,
  Iuml     => 207,
  iuml     => 239,
  Kappa    => 922,
  kappa    => 954,
  Lambda   => 923,
  lambda   => 955,
  lang     => 9001,
  laquo    => 171,
  lArr     => 8656,
  larr     => 8592,
  lceil    => 8968,
  ldquo    => 8220,
  le       => 8804,
  lfloor   => 8970,
  lowast   => 8727,
  loz      => 9674,
  lrm      => 8206,
  lsaquo   => 8249,
  lsquo    => 8216,
  lt       => 60,
  macr     => 175,
  mdash    => 8212,
  micro    => 181,
  middot   => 183,
  minus    => 8722,
  Mu       => 924,
  mu       => 956,
  nabla    => 8711,
  nbsp     => 160,
  ndash    => 8211,
  ne       => 8800,
  ni       => 8715,
  not      => 172,
  notin    => 8713,
  nsub     => 8836,
  Ntilde   => 209,
  ntilde   => 241,
  Nu       => 925,
  nu       => 957,
  Oacute   => 211,
  oacute   => 243,
  Ocirc    => 212,
  ocirc    => 244,
  OElig    => 338,
  oelig    => 339,
  Ograve   => 210,
  ograve   => 242,
  oline    => 8254,
  Omega    => 937,
  omega    => 969,
  Omicron  => 927,
  omicron  => 959,
  oplus    => 8853,
  or       => 8744,
  ordf     => 170,
  ordm     => 186,
  Oslash   => 216,
  oslash   => 248,
  Otilde   => 213,
  otilde   => 245,
  otimes   => 8855,
  Ouml     => 214,
  ouml     => 246,
  para     => 182,
  part     => 8706,
  permil   => 8240,
  perp     => 8869,
  Phi      => 934,
  phi      => 966,
  Pi       => 928,
  pi       => 960,
  piv      => 982,
  plusmn   => 177,
  pound    => 163,
  Prime    => 8243,
  prime    => 8242,
  prod     => 8719,
  prop     => 8733,
  Psi      => 936,
  psi      => 968,
  quot     => 34,
  radic    => 8730,
  rang     => 9002,
  raquo    => 187,
  rArr     => 8658,
  rarr     => 8594,
  rceil    => 8969,
  rdquo    => 8221,
  real     => 8476,
  reg      => 174,
  rfloor   => 8971,
  Rho      => 929,
  rho      => 961,
  rlm      => 8207,
  rsaquo   => 8250,
  rsquo    => 8217,
  sbquo    => 8218,
  Scaron   => 352,
  scaron   => 353,
  sdot     => 8901,
  sect     => 167,
  shy      => 173,
  Sigma    => 931,
  sigma    => 963,
  sigmaf   => 962,
  sim      => 8764,
  spades   => 9824,
  sub      => 8834,
  sube     => 8838,
  sum      => 8721,
  sup      => 8835,
  sup1     => 185,
  sup2     => 178,
  sup3     => 179,
  supe     => 8839,
  szlig    => 223,
  Tau      => 932,
  tau      => 964,
  there4   => 8756,
  Theta    => 920,
  theta    => 952,
  thetasym => 977,
  thinsp   => 8201,
  THORN    => 222,
  thorn    => 254,
  tilde    => 732,
  times    => 215,
  trade    => 8482,
  Uacute   => 218,
  uacute   => 250,
  uArr     => 8657,
  uarr     => 8593,
  Ucirc    => 219,
  ucirc    => 251,
  Ugrave   => 217,
  ugrave   => 249,
  uml      => 168,
  upsih    => 978,
  Upsilon  => 933,
  upsilon  => 965,
  Uuml     => 220,
  uuml     => 252,
  weierp   => 8472,
  Xi       => 926,
  xi       => 958,
  Yacute   => 221,
  yacute   => 253,
  yen      => 165,
  Yuml     => 376,
  yuml     => 255,
  Zeta     => 918,
  zeta     => 950,
  zwj      => 8205,
  zwnj     => 8204
);

# Reverse entities for html_escape
my %REVERSE_ENTITIES = reverse %ENTITIES;

# "apos"
$ENTITIES{apos} = 39;

# Encode cache
my %ENCODE;

# "Bart, stop pestering Satan!"
our @EXPORT_OK = qw/b64_decode b64_encode camelize decamelize decode encode/;
push @EXPORT_OK, qw/get_line hmac_md5_sum hmac_sha1_sum html_escape/;
push @EXPORT_OK, qw/html_unescape md5_bytes md5_sum punycode_decode/;
push @EXPORT_OK, qw/punycode_encode qp_decode qp_encode quote/;
push @EXPORT_OK, qw/secure_compare sha1_bytes sha1_sum trim unquote/;
push @EXPORT_OK, qw/url_escape url_unescape xml_escape/;

sub b64_decode { MIME::Base64::decode_base64(@_) }

sub b64_encode { MIME::Base64::encode_base64(@_) }

sub camelize {
  my $string = shift;
  return $string if $string =~ /^[A-Z]/;

  # Module parts
  my @parts;
  for my $part (split /-/, $string) {
    next unless $part;

    # Camel case words
    my @words = split /_/, $part;
    @words = map { ucfirst lc } @words;
    push @parts, join '', @words;
  }
  return join '::', @parts;
}

sub decamelize {
  my $string = shift;
  return $string if $string !~ /^[A-Z]/;

  # Module parts
  my @parts;
  for my $part (split /\:\:/, $string) {

    # Camel case words
    my @words;
    push @words, $1 while ($part =~ s/([A-Z]{1}[^A-Z]*)//);
    @words = map {lc} @words;
    push @parts, join '_', @words;
  }
  return join '-', @parts;
}

sub decode {
  my ($encoding, $bytes) = @_;

  # Try decoding
  return unless eval {

    # UTF-8
    if ($encoding eq 'UTF-8') { die unless utf8::decode $bytes }

    # Everything else
    else {
      $bytes =
        ($ENCODE{$encoding} ||= Encode::find_encoding($encoding))
        ->decode($bytes, 1);
    }

    1;
  };

  return $bytes;
}

sub encode {
  my ($encoding, $chars) = @_;

  # UTF-8
  if ($encoding eq 'UTF-8') {
    utf8::encode $chars;
    return $chars;
  }

  # Everything else
  return ($ENCODE{$encoding} ||= Encode::find_encoding($encoding))
    ->encode($chars);
}

sub get_line {

  # Locate line ending
  return if (my $pos = index $_[0], "\x0a") == -1;

  # Extract line and ending
  my $line = substr $_[0], 0, $pos + 1, '';
  $line =~ s/\x0d?\x0a$//;

  return $line;
}

sub hmac_md5_sum { _hmac(\&_md5, @_) }

sub hmac_sha1_sum { _hmac(\&_sha1, @_) }

sub html_escape {
  my $string  = shift;
  my $escaped = '';
  for (1 .. length $string) {

    # Escape entities
    my $char = substr $string, 0, 1, '';
    my $num = unpack 'U', $char;
    my $named = $REVERSE_ENTITIES{$num};
    $char = "&$named;" if $named;
    $escaped .= $char;
  }
  return $escaped;
}

# "Daddy, I'm scared. Too scared to even wet my pants.
#  Just relax and it'll come, son."
sub html_unescape {
  my $string = shift;
  $string =~ s/
    &
    (?:
      \#
      (
        (?:
          \d{1,7}             # Number
          |
          x[0-9A-Fa-f]{1,6}   # Hex
        )
      )
      |
      ([A-Za-z]{1,8})         # Name
    )
    ;
  /_unescape($1, $2)/gex;
  return $string;
}

sub md5_bytes { _md5(@_) }

sub md5_sum { Digest::MD5::md5_hex(@_) }

sub punycode_decode {
  my $input = shift;
  use integer;

  # Defaults
  my $n    = PUNYCODE_INITIAL_N;
  my $i    = 0;
  my $bias = PUNYCODE_INITIAL_BIAS;
  my @output;

  # Delimiter
  if ($input =~ s/(.*)$DELIMITER//s) { push @output, split //, $1 }

  # Decode (direct translation of RFC 3492)
  while (length $input) {
    my $oldi = $i;
    my $w    = 1;

    # Base to infinity in steps of base
    for (my $k = PUNYCODE_BASE; 1; $k += PUNYCODE_BASE) {

      # Digit
      my $digit = ord substr $input, 0, 1, '';
      $digit = $digit < 0x40 ? $digit + (26 - 0x30) : ($digit & 0x1f) - 1;
      $i += $digit * $w;
      my $t = $k - $bias;
      $t =
          $t < PUNYCODE_TMIN ? PUNYCODE_TMIN
        : $t > PUNYCODE_TMAX ? PUNYCODE_TMAX
        :                      $t;
      last if $digit < $t;

      $w *= (PUNYCODE_BASE - $t);
    }

    # Bias
    $bias = _adapt($i - $oldi, @output + 1, $oldi == 0);
    $n += $i / (@output + 1);
    $i = $i % (@output + 1);

    # Insert
    splice @output, $i, 0, chr($n);
    $i++;
  }

  return join '', @output;
}

sub punycode_encode {
  use integer;

  # Defaults
  my $output = shift;
  my $len    = length $output;

  # Split input
  my @input = map ord, split //, $output;
  my @chars = sort grep { $_ >= PUNYCODE_INITIAL_N } @input;

  # Remove non basic characters
  $output =~ s/[^\x00-\x7f]+//gs;

  # Non basic characters in input
  my $h = my $b = length $output;
  $output .= $DELIMITER if $b > 0;

  # Defaults
  my $n     = PUNYCODE_INITIAL_N;
  my $delta = 0;
  my $bias  = PUNYCODE_INITIAL_BIAS;

  # Encode (direct translation of RFC 3492)
  for my $m (@chars) {

    # Basic character
    next if $m < $n;

    # Delta
    $delta += ($m - $n) * ($h + 1);

    # Walk all code points in order
    $n = $m;
    for (my $i = 0; $i < $len; $i++) {
      my $c = $input[$i];

      # Basic character
      $delta++ if $c < $n;

      # Non basic character
      if ($c == $n) {
        my $q = $delta;

        # Base to infinity in steps of base
        for (my $k = PUNYCODE_BASE; 1; $k += PUNYCODE_BASE) {
          my $t = $k - $bias;
          $t =
              $t < PUNYCODE_TMIN ? PUNYCODE_TMIN
            : $t > PUNYCODE_TMAX ? PUNYCODE_TMAX
            :                      $t;
          last if $q < $t;

          # Code point for digit "t"
          my $o = $t + (($q - $t) % (PUNYCODE_BASE - $t));
          $output .= chr $o + ($o < 26 ? 0x61 : 0x30 - 26);

          $q = ($q - $t) / (PUNYCODE_BASE - $t);
        }

        # Code point for digit "q"
        $output .= chr $q + ($q < 26 ? 0x61 : 0x30 - 26);

        # Bias
        $bias = _adapt($delta, $h + 1, $h == $b);
        $delta = 0;
        $h++;
      }
    }

    $delta++;
    $n++;
  }

  return $output;
}

sub qp_decode { MIME::QuotedPrint::decode_qp(@_) }

sub qp_encode { MIME::QuotedPrint::encode_qp(@_) }

sub quote {
  my $string = shift;
  $string =~ s/(["\\])/\\$1/g;
  return qq/"$string"/;
}

sub secure_compare {
  my ($a, $b) = @_;
  return if length $a != length $b;
  my $r = 0;
  $r |= ord(substr $a, $_) ^ ord(substr $b, $_) for 0 .. length($a) - 1;
  return $r == 0 ? 1 : undef;
}

sub sha1_bytes { _sha1(@_) }

sub sha1_sum { Digest::SHA::sha1_hex(@_) }

sub trim {
  my $string = shift;
  for ($string) {
    s/^\s*//;
    s/\s*$//;
  }
  return $string;
}

sub unquote {
  my $string = shift;
  return $string unless $string =~ /^".*"$/g;

  # Unquote
  for ($string) {
    s/^"//g;
    s/"$//g;
    s/\\\\/\\/g;
    s/\\"/"/g;
  }

  return $string;
}

sub url_escape {
  my ($string, $pattern) = @_;
  $pattern ||= 'A-Za-z0-9\-\.\_\~';
  return $string unless $string =~ /[^$pattern]/;
  $string =~ s/([^$pattern])/sprintf('%%%02X',ord($1))/ge;
  return $string;
}

# "I've gone back in time to when dinosaurs weren't just confined to zoos."
sub url_unescape {
  my $string = shift;
  return $string if index($string, '%') == -1;
  $string =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/ge;
  return $string;
}

sub xml_escape {
  my $string = shift;
  for ($string) {
    s/&/&amp;/g;
    s/</&lt;/g;
    s/>/&gt;/g;
    s/"/&quot;/g;
    s/'/&#39;/g;
  }
  return $string;
}

# Helper for punycode
sub _adapt {
  my ($delta, $numpoints, $firsttime) = @_;

  use integer;
  $delta = $firsttime ? $delta / PUNYCODE_DAMP : $delta / 2;
  $delta += $delta / $numpoints;
  my $k = 0;
  while ($delta > ((PUNYCODE_BASE - PUNYCODE_TMIN) * PUNYCODE_TMAX) / 2) {
    $delta /= PUNYCODE_BASE - PUNYCODE_TMIN;
    $k += PUNYCODE_BASE;
  }

  return $k
    + (
    ((PUNYCODE_BASE - PUNYCODE_TMIN + 1) * $delta) / ($delta + PUNYCODE_SKEW)
    );
}

sub _hmac {

  # Secret
  my $secret = $_[2] || 'Very unsecure!';
  $secret = $_[0]->($secret) if length $secret > 64;<commit_msg>shorter constant names<commit_after>package Mojo::Util;
use Mojo::Base 'Exporter';

require Digest::MD5;
require Digest::SHA;
require Encode;
require MIME::Base64;
require MIME::QuotedPrint;

# Punycode bootstring parameters
use constant {
  PC_BASE         => 36,
  PC_TMIN         => 1,
  PC_TMAX         => 26,
  PC_SKEW         => 38,
  PC_DAMP         => 700,
  PC_INITIAL_BIAS => 72,
  PC_INITIAL_N    => 128
};

# Punycode delimiter
my $DELIMITER = chr 0x2D;

# HTML5 entities for html_unescape (without "apos")
my %ENTITIES = (
  Aacute   => 193,
  aacute   => 225,
  Acirc    => 194,
  acirc    => 226,
  acute    => 180,
  AElig    => 198,
  aelig    => 230,
  Agrave   => 192,
  agrave   => 224,
  alefsym  => 8501,
  Alpha    => 913,
  alpha    => 945,
  amp      => 38,
  and      => 8743,
  ang      => 8736,
  '#39'    => 39,
  Aring    => 197,
  aring    => 229,
  asymp    => 8776,
  Atilde   => 195,
  atilde   => 227,
  Auml     => 196,
  auml     => 228,
  bdquo    => 8222,
  Beta     => 914,
  beta     => 946,
  brvbar   => 166,
  bull     => 8226,
  cap      => 8745,
  Ccedil   => 199,
  ccedil   => 231,
  cedil    => 184,
  cent     => 162,
  Chi      => 935,
  chi      => 967,
  circ     => 710,
  clubs    => 9827,
  cong     => 8773,
  copy     => 169,
  crarr    => 8629,
  cup      => 8746,
  curren   => 164,
  Dagger   => 8225,
  dagger   => 8224,
  dArr     => 8659,
  darr     => 8595,
  deg      => 176,
  Delta    => 916,
  delta    => 948,
  diams    => 9830,
  divide   => 247,
  Eacute   => 201,
  eacute   => 233,
  Ecirc    => 202,
  ecirc    => 234,
  Egrave   => 200,
  egrave   => 232,
  empty    => 8709,
  emsp     => 8195,
  ensp     => 8194,
  Epsilon  => 917,
  epsilon  => 949,
  equiv    => 8801,
  Eta      => 919,
  eta      => 951,
  ETH      => 208,
  eth      => 240,
  Euml     => 203,
  euml     => 235,
  euro     => 8364,
  exist    => 8707,
  fnof     => 402,
  forall   => 8704,
  frac12   => 189,
  frac14   => 188,
  frac34   => 190,
  frasl    => 8260,
  Gamma    => 915,
  gamma    => 947,
  ge       => 8805,
  gt       => 62,
  hArr     => 8660,
  harr     => 8596,
  hearts   => 9829,
  hellip   => 8230,
  Iacute   => 205,
  iacute   => 237,
  Icirc    => 206,
  icirc    => 238,
  iexcl    => 161,
  Igrave   => 204,
  igrave   => 236,
  image    => 8465,
  infin    => 8734,
  int      => 8747,
  Iota     => 921,
  iota     => 953,
  iquest   => 191,
  isin     => 8712,
  Iuml     => 207,
  iuml     => 239,
  Kappa    => 922,
  kappa    => 954,
  Lambda   => 923,
  lambda   => 955,
  lang     => 9001,
  laquo    => 171,
  lArr     => 8656,
  larr     => 8592,
  lceil    => 8968,
  ldquo    => 8220,
  le       => 8804,
  lfloor   => 8970,
  lowast   => 8727,
  loz      => 9674,
  lrm      => 8206,
  lsaquo   => 8249,
  lsquo    => 8216,
  lt       => 60,
  macr     => 175,
  mdash    => 8212,
  micro    => 181,
  middot   => 183,
  minus    => 8722,
  Mu       => 924,
  mu       => 956,
  nabla    => 8711,
  nbsp     => 160,
  ndash    => 8211,
  ne       => 8800,
  ni       => 8715,
  not      => 172,
  notin    => 8713,
  nsub     => 8836,
  Ntilde   => 209,
  ntilde   => 241,
  Nu       => 925,
  nu       => 957,
  Oacute   => 211,
  oacute   => 243,
  Ocirc    => 212,
  ocirc    => 244,
  OElig    => 338,
  oelig    => 339,
  Ograve   => 210,
  ograve   => 242,
  oline    => 8254,
  Omega    => 937,
  omega    => 969,
  Omicron  => 927,
  omicron  => 959,
  oplus    => 8853,
  or       => 8744,
  ordf     => 170,
  ordm     => 186,
  Oslash   => 216,
  oslash   => 248,
  Otilde   => 213,
  otilde   => 245,
  otimes   => 8855,
  Ouml     => 214,
  ouml     => 246,
  para     => 182,
  part     => 8706,
  permil   => 8240,
  perp     => 8869,
  Phi      => 934,
  phi      => 966,
  Pi       => 928,
  pi       => 960,
  piv      => 982,
  plusmn   => 177,
  pound    => 163,
  Prime    => 8243,
  prime    => 8242,
  prod     => 8719,
  prop     => 8733,
  Psi      => 936,
  psi      => 968,
  quot     => 34,
  radic    => 8730,
  rang     => 9002,
  raquo    => 187,
  rArr     => 8658,
  rarr     => 8594,
  rceil    => 8969,
  rdquo    => 8221,
  real     => 8476,
  reg      => 174,
  rfloor   => 8971,
  Rho      => 929,
  rho      => 961,
  rlm      => 8207,
  rsaquo   => 8250,
  rsquo    => 8217,
  sbquo    => 8218,
  Scaron   => 352,
  scaron   => 353,
  sdot     => 8901,
  sect     => 167,
  shy      => 173,
  Sigma    => 931,
  sigma    => 963,
  sigmaf   => 962,
  sim      => 8764,
  spades   => 9824,
  sub      => 8834,
  sube     => 8838,
  sum      => 8721,
  sup      => 8835,
  sup1     => 185,
  sup2     => 178,
  sup3     => 179,
  supe     => 8839,
  szlig    => 223,
  Tau      => 932,
  tau      => 964,
  there4   => 8756,
  Theta    => 920,
  theta    => 952,
  thetasym => 977,
  thinsp   => 8201,
  THORN    => 222,
  thorn    => 254,
  tilde    => 732,
  times    => 215,
  trade    => 8482,
  Uacute   => 218,
  uacute   => 250,
  uArr     => 8657,
  uarr     => 8593,
  Ucirc    => 219,
  ucirc    => 251,
  Ugrave   => 217,
  ugrave   => 249,
  uml      => 168,
  upsih    => 978,
  Upsilon  => 933,
  upsilon  => 965,
  Uuml     => 220,
  uuml     => 252,
  weierp   => 8472,
  Xi       => 926,
  xi       => 958,
  Yacute   => 221,
  yacute   => 253,
  yen      => 165,
  Yuml     => 376,
  yuml     => 255,
  Zeta     => 918,
  zeta     => 950,
  zwj      => 8205,
  zwnj     => 8204
);

# Reverse entities for html_escape
my %REVERSE_ENTITIES = reverse %ENTITIES;

# "apos"
$ENTITIES{apos} = 39;

# Encode cache
my %ENCODE;

# "Bart, stop pestering Satan!"
our @EXPORT_OK = qw/b64_decode b64_encode camelize decamelize decode encode/;
push @EXPORT_OK, qw/get_line hmac_md5_sum hmac_sha1_sum html_escape/;
push @EXPORT_OK, qw/html_unescape md5_bytes md5_sum punycode_decode/;
push @EXPORT_OK, qw/punycode_encode qp_decode qp_encode quote/;
push @EXPORT_OK, qw/secure_compare sha1_bytes sha1_sum trim unquote/;
push @EXPORT_OK, qw/url_escape url_unescape xml_escape/;

sub b64_decode { MIME::Base64::decode_base64(@_) }

sub b64_encode { MIME::Base64::encode_base64(@_) }

sub camelize {
  my $string = shift;
  return $string if $string =~ /^[A-Z]/;

  # Module parts
  my @parts;
  for my $part (split /-/, $string) {
    next unless $part;

    # Camel case words
    my @words = split /_/, $part;
    @words = map { ucfirst lc } @words;
    push @parts, join '', @words;
  }
  return join '::', @parts;
}

sub decamelize {
  my $string = shift;
  return $string if $string !~ /^[A-Z]/;

  # Module parts
  my @parts;
  for my $part (split /\:\:/, $string) {

    # Camel case words
    my @words;
    push @words, $1 while ($part =~ s/([A-Z]{1}[^A-Z]*)//);
    @words = map {lc} @words;
    push @parts, join '_', @words;
  }
  return join '-', @parts;
}

sub decode {
  my ($encoding, $bytes) = @_;

  # Try decoding
  return unless eval {

    # UTF-8
    if ($encoding eq 'UTF-8') { die unless utf8::decode $bytes }

    # Everything else
    else {
      $bytes =
        ($ENCODE{$encoding} ||= Encode::find_encoding($encoding))
        ->decode($bytes, 1);
    }

    1;
  };

  return $bytes;
}

sub encode {
  my ($encoding, $chars) = @_;

  # UTF-8
  if ($encoding eq 'UTF-8') {
    utf8::encode $chars;
    return $chars;
  }

  # Everything else
  return ($ENCODE{$encoding} ||= Encode::find_encoding($encoding))
    ->encode($chars);
}

sub get_line {

  # Locate line ending
  return if (my $pos = index $_[0], "\x0a") == -1;

  # Extract line and ending
  my $line = substr $_[0], 0, $pos + 1, '';
  $line =~ s/\x0d?\x0a$//;

  return $line;
}

sub hmac_md5_sum { _hmac(\&_md5, @_) }

sub hmac_sha1_sum { _hmac(\&_sha1, @_) }

sub html_escape {
  my $string  = shift;
  my $escaped = '';
  for (1 .. length $string) {

    # Escape entities
    my $char = substr $string, 0, 1, '';
    my $num = unpack 'U', $char;
    my $named = $REVERSE_ENTITIES{$num};
    $char = "&$named;" if $named;
    $escaped .= $char;
  }
  return $escaped;
}

# "Daddy, I'm scared. Too scared to even wet my pants.
#  Just relax and it'll come, son."
sub html_unescape {
  my $string = shift;
  $string =~ s/
    &
    (?:
      \#
      (
        (?:
          \d{1,7}             # Number
          |
          x[0-9A-Fa-f]{1,6}   # Hex
        )
      )
      |
      ([A-Za-z]{1,8})         # Name
    )
    ;
  /_unescape($1, $2)/gex;
  return $string;
}

sub md5_bytes { _md5(@_) }

sub md5_sum { Digest::MD5::md5_hex(@_) }

sub punycode_decode {
  my $input = shift;
  use integer;

  # Defaults
  my $n    = PC_INITIAL_N;
  my $i    = 0;
  my $bias = PC_INITIAL_BIAS;
  my @output;

  # Delimiter
  if ($input =~ s/(.*)$DELIMITER//s) { push @output, split //, $1 }

  # Decode (direct translation of RFC 3492)
  while (length $input) {
    my $oldi = $i;
    my $w    = 1;

    # Base to infinity in steps of base
    for (my $k = PC_BASE; 1; $k += PC_BASE) {

      # Digit
      my $digit = ord substr $input, 0, 1, '';
      $digit = $digit < 0x40 ? $digit + (26 - 0x30) : ($digit & 0x1f) - 1;
      $i += $digit * $w;
      my $t = $k - $bias;
      $t = $t < PC_TMIN ? PC_TMIN : $t > PC_TMAX ? PC_TMAX : $t;
      last if $digit < $t;

      $w *= (PC_BASE - $t);
    }

    # Bias
    $bias = _adapt($i - $oldi, @output + 1, $oldi == 0);
    $n += $i / (@output + 1);
    $i = $i % (@output + 1);

    # Insert
    splice @output, $i, 0, chr($n);
    $i++;
  }

  return join '', @output;
}

sub punycode_encode {
  use integer;

  # Defaults
  my $output = shift;
  my $len    = length $output;

  # Split input
  my @input = map ord, split //, $output;
  my @chars = sort grep { $_ >= PC_INITIAL_N } @input;

  # Remove non basic characters
  $output =~ s/[^\x00-\x7f]+//gs;

  # Non basic characters in input
  my $h = my $b = length $output;
  $output .= $DELIMITER if $b > 0;

  # Defaults
  my $n     = PC_INITIAL_N;
  my $delta = 0;
  my $bias  = PC_INITIAL_BIAS;

  # Encode (direct translation of RFC 3492)
  for my $m (@chars) {

    # Basic character
    next if $m < $n;

    # Delta
    $delta += ($m - $n) * ($h + 1);

    # Walk all code points in order
    $n = $m;
    for (my $i = 0; $i < $len; $i++) {
      my $c = $input[$i];

      # Basic character
      $delta++ if $c < $n;

      # Non basic character
      if ($c == $n) {
        my $q = $delta;

        # Base to infinity in steps of base
        for (my $k = PC_BASE; 1; $k += PC_BASE) {
          my $t = $k - $bias;
          $t = $t < PC_TMIN ? PC_TMIN : $t > PC_TMAX ? PC_TMAX : $t;
          last if $q < $t;

          # Code point for digit "t"
          my $o = $t + (($q - $t) % (PC_BASE - $t));
          $output .= chr $o + ($o < 26 ? 0x61 : 0x30 - 26);

          $q = ($q - $t) / (PC_BASE - $t);
        }

        # Code point for digit "q"
        $output .= chr $q + ($q < 26 ? 0x61 : 0x30 - 26);

        # Bias
        $bias = _adapt($delta, $h + 1, $h == $b);
        $delta = 0;
        $h++;
      }
    }

    $delta++;
    $n++;
  }

  return $output;
}

sub qp_decode { MIME::QuotedPrint::decode_qp(@_) }

sub qp_encode { MIME::QuotedPrint::encode_qp(@_) }

sub quote {
  my $string = shift;
  $string =~ s/(["\\])/\\$1/g;
  return qq/"$string"/;
}

sub secure_compare {
  my ($a, $b) = @_;
  return if length $a != length $b;
  my $r = 0;
  $r |= ord(substr $a, $_) ^ ord(substr $b, $_) for 0 .. length($a) - 1;
  return $r == 0 ? 1 : undef;
}

sub sha1_bytes { _sha1(@_) }

sub sha1_sum { Digest::SHA::sha1_hex(@_) }

sub trim {
  my $string = shift;
  for ($string) {
    s/^\s*//;
    s/\s*$//;
  }
  return $string;
}

sub unquote {
  my $string = shift;
  return $string unless $string =~ /^".*"$/g;

  # Unquote
  for ($string) {
    s/^"//g;
    s/"$//g;
    s/\\\\/\\/g;
    s/\\"/"/g;
  }

  return $string;
}

sub url_escape {
  my ($string, $pattern) = @_;
  $pattern ||= 'A-Za-z0-9\-\.\_\~';
  return $string unless $string =~ /[^$pattern]/;
  $string =~ s/([^$pattern])/sprintf('%%%02X',ord($1))/ge;
  return $string;
}

# "I've gone back in time to when dinosaurs weren't just confined to zoos."
sub url_unescape {
  my $string = shift;
  return $string if index($string, '%') == -1;
  $string =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/ge;
  return $string;
}

sub xml_escape {
  my $string = shift;
  for ($string) {
    s/&/&amp;/g;
    s/</&lt;/g;
    s/>/&gt;/g;
    s/"/&quot;/g;
    s/'/&#39;/g;
  }
  return $string;
}

# Helper for punycode
sub _adapt {
  my ($delta, $numpoints, $firsttime) = @_;

  use integer;
  $delta = $firsttime ? $delta / PC_DAMP : $delta / 2;
  $delta += $delta / $numpoints;
  my $k = 0;
  while ($delta > ((PC_BASE - PC_TMIN) * PC_TMAX) / 2) {
    $delta /= PC_BASE - PC_TMIN;
    $k += PC_BASE;
  }

  return $k + (((PC_BASE - PC_TMIN + 1) * $delta) / ($delta + PC_SKEW));
}

sub _hmac {

  # Secret
  my $secret = $_[2] || 'Very unsecure!';
  $secret = $_[0]->($secret) if length $secret > 64;
{'fraction_non_alphanumeric': 0.17120116328014387, 'fraction_numerical': 0.07905713083075039, 'mean_word_length': 1.8391091797935903, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 543, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1627953', 'n_tokens_mistral': 12769, 'n_tokens_neox': 10771, 'n_words': 2731}