<commit_before>package Mojolicious::Command::Get;
use Mojo::Base 'Mojo::Command';

use Mojo::Client;
use Mojo::DOM;
use Mojo::IOLoop;
use Mojo::Transaction::HTTP;
use Mojo::Util 'decode';

use Getopt::Long 'GetOptions';

has description => <<'EOF';
Get file from URL.
EOF
has usage => <<"EOF";
usage: $0 get [OPTIONS] [URL] [SELECTOR] [COMMANDS]

  mojo get /
  mojo get mojolicio.us
  mojo get -v -r google.com
  mojo get --method POST --content 'trololo' mojolicio.us
  mojo get --header 'X-Bender: Bite my shiny metal ass!' mojolicio.us
  mojo get mojolicio.us 'head > title' text
  mojo get mojolicio.us .footer all
  mojo get mojolicio.us a attr href
  mojo get mojolicio.us '*' attr id
  mojo get mojolicio.us 'h1, h2, h3' 3 text

These options are available:
  --header     Additional HTTP header.
  --method     HTTP method to use.
  --redirect   Follow up to 5 redirects.
  --verbose    Print verbose debug information to STDERR.
EOF

# "I hope this has taught you kids a lesson: kids never learn."
sub run {
  my $self = shift;

  # Options
  local @ARGV = @_ if @_;
  my $method = 'GET';
  my @headers;
  my $content = '';
  my ($redirect, $verbose) = 0;
  GetOptions(
    'content=s' => sub { $content  = $_[1] },
    'header=s'  => \@headers,
    'method=s'  => sub { $method   = $_[1] },
    'redirect'  => sub { $redirect = 1 },
    'verbose'   => sub { $verbose  = 1 }
  );

  # Headers
  my $headers = {};
  for my $header (@headers) {
    next unless $header =~ /^\s*([^\:]+)\s*:\s*([^\:]+)\s*$/;
    $headers->{$1} = $2;
  }

  # URL
  my $url = shift @ARGV;
  die $self->usage unless $url;
  decode 'UTF-8', $url;

  # Selector
  my $selector = shift @ARGV;

  # Buffer
  my $buffer = '';

  # Client
  my $client = Mojo::Client->new(ioloop => Mojo::IOLoop->singleton);

  # Silence
  $client->log->level('fatal');

  # Absolute URL
  if ($url =~ /^\w+:\/\//) { $client->detect_proxy }

  # Application
  else { $client->app($ENV{MOJO_APP} || 'Mojo::HelloWorld') }

  # Follow redirects
  $client->max_redirects(5) if $redirect;

  # Start
  my $v;
  $client->on_start(
    sub {
      my $tx = pop;

      # Prepare request information
      my $req       = $tx->req;
      my $startline = $req->build_start_line;
      my $headers   = $req->build_headers;

      # Verbose callback
      my $v  = $verbose;
      my $cb = sub {
        my $res = shift;

        # Wait for headers
        return unless $v && $res->headers->is_done;

        # Request
        warn "$startline$headers";

        # Response
        my $version = $res->version;
        my $code    = $res->code;
        my $message = $res->message;
        warn "HTTP/$version $code $message\n",
          $res->headers->to_string, "\n\n";

        # Done
        $v = 0;
      };

      # Progress
      $tx->res->on_progress(sub { $cb->(shift) });

      # Stream content
      $tx->res->body(
        sub {
          $cb->(my $res = shift);

          # Ignore intermediate content
          return if $redirect && $res->is_status_class(300);

          # Chunk
          $selector ? ($buffer .= pop) : print(pop);
        }
      );
    }
  );

  # Transaction
  my $tx = $client->build_tx($method, $url, $headers, $content);

  # Get
  $client->start($tx);

  # Error
  my ($message, $code) = $tx->error;
  warn qq/Problem loading URL "$url". ($message)\n/ if $message && !$code;

  # Charset
  my $charset = 'UTF-8';
  ($tx->res->headers->content_type || '') =~ /charset=\"?([^\"\s;]+)\"?/
    and $charset = $1;

  # Select
  $self->_select($buffer, $charset, $selector) if $selector;
<commit_msg>fixed small get command bug<commit_after>package Mojolicious::Command::Get;
use Mojo::Base 'Mojo::Command';

use Mojo::Client;
use Mojo::DOM;
use Mojo::IOLoop;
use Mojo::Transaction::HTTP;
use Mojo::Util 'decode';

use Getopt::Long 'GetOptions';

has description => <<'EOF';
Get file from URL.
EOF
has usage => <<"EOF";
usage: $0 get [OPTIONS] [URL] [SELECTOR] [COMMANDS]

  mojo get /
  mojo get mojolicio.us
  mojo get -v -r google.com
  mojo get --method POST --content 'trololo' mojolicio.us
  mojo get --header 'X-Bender: Bite my shiny metal ass!' mojolicio.us
  mojo get mojolicio.us 'head > title' text
  mojo get mojolicio.us .footer all
  mojo get mojolicio.us a attr href
  mojo get mojolicio.us '*' attr id
  mojo get mojolicio.us 'h1, h2, h3' 3 text

These options are available:
  --header     Additional HTTP header.
  --method     HTTP method to use.
  --redirect   Follow up to 5 redirects.
  --verbose    Print verbose debug information to STDERR.
EOF

# "I hope this has taught you kids a lesson: kids never learn."
sub run {
  my $self = shift;

  # Options
  local @ARGV = @_ if @_;
  my $method = 'GET';
  my @headers;
  my $content = '';
  my ($redirect, $verbose) = 0;
  GetOptions(
    'content=s' => sub { $content  = $_[1] },
    'header=s'  => \@headers,
    'method=s'  => sub { $method   = $_[1] },
    'redirect'  => sub { $redirect = 1 },
    'verbose'   => sub { $verbose  = 1 }
  );

  # Headers
  my $headers = {};
  for my $header (@headers) {
    next unless $header =~ /^\s*([^\:]+)\s*:\s*([^\:]+)\s*$/;
    $headers->{$1} = $2;
  }

  # URL
  my $url = shift @ARGV;
  die $self->usage unless $url;
  decode 'UTF-8', $url;

  # Selector
  my $selector = shift @ARGV;

  # Buffer
  my $buffer = '';

  # Client
  my $client = Mojo::Client->new(ioloop => Mojo::IOLoop->singleton);

  # Silence
  $client->log->level('fatal');

  # Absolute URL
  if ($url =~ /^\w+:\/\//) { $client->detect_proxy }

  # Application
  else { $client->app($ENV{MOJO_APP} || 'Mojo::HelloWorld') }

  # Follow redirects
  $client->max_redirects(5) if $redirect;

  # Start
  my $v;
  $client->on_start(
    sub {
      my $tx = pop;

      # Prepare request information
      my $req       = $tx->req;
      my $startline = $req->build_start_line;
      my $headers   = $req->build_headers;

      # Verbose callback
      my $v  = $verbose;
      my $cb = sub {
        my $res = shift;

        # Wait for headers
        return unless $v && $res->headers->is_done;

        # Request
        warn "$startline$headers";

        # Response
        my $version = $res->version;
        my $code    = $res->code;
        my $message = $res->message;
        warn "HTTP/$version $code $message\n",
          $res->headers->to_string, "\n\n";

        # Done
        $v = 0;
      };

      # Progress
      $tx->res->on_progress(sub { $cb->(shift) });

      # Stream content
      $tx->res->body(
        sub {
          $cb->(my $res = shift);

          # Ignore intermediate content
          return if $redirect && $res->is_status_class(300);

          # Chunk
          $selector ? ($buffer .= pop) : print(pop);
        }
      );
    }
  );

  # Transaction
  my $tx = $client->build_tx($method, $url, $headers, $content);

  # Default charset
  my $charset = 'UTF-8';

  # Get
  $client->start(
    $tx => sub {
      my $tx = pop;

      # Error
      my ($message, $code) = $tx->error;
      warn qq/Problem loading URL "$url". ($message)\n/ if $message && !$code;

      # Charset
      ($tx->res->headers->content_type || '') =~ /charset=\"?([^\"\s;]+)\"?/
        and $charset = $1;
    }
  );

  # Select
  $self->_select($buffer, $charset, $selector) if $selector;

{'fraction_non_alphanumeric': 0.17265989569036508, 'fraction_numerical': 0.005764479824320615, 'mean_word_length': 2.3908794788273617, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2621578', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2578, 'n_words': 705}