<commit_before># Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Mechanize;
use WWW::Mechanize;
use LWP::Protocol::https;
use Mozilla::CA;
use IO::Socket::SSL;
use FlashVideo::Downloader;
use Encode ();

use strict;
use base "WWW::Mechanize";

sub new {
  my $class = shift;
  my $browser = $class->SUPER::new(autocheck => 0);
  $browser->agent_alias("Windows Mozilla");

  my $proxy = $App::get_flash_videos::opt{proxy};

  if ($proxy) {
    if ($proxy =~ /^(\w+):?(\d+)?$/) {
      # Proxy is in format:
      #   localhost:1337
      #   localhost
      # Add a scheme so LWP can use it.
      # Other formats are passed to LWP directly.
      my ($host, $port) = ($1, $2);

      $port ||= 1080; # socks by default

      $proxy = "socks://$host:$port";

    }
    print STDERR "Using proxy server $proxy\n"
      if $App::get_flash_videos::opt{debug};

    $browser->proxy([qw[http https]] => $proxy);<commit_msg>Only add LWP::Protcol::https combined file getting too large<commit_after># Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Mechanize;
use WWW::Mechanize;
use LWP::Protocol::https;
use FlashVideo::Downloader;
use Encode ();

use strict;
use base "WWW::Mechanize";

sub new {
  my $class = shift;
  my $browser = $class->SUPER::new(autocheck => 0);
  $browser->agent_alias("Windows Mozilla");

  my $proxy = $App::get_flash_videos::opt{proxy};

  if ($proxy) {
    if ($proxy =~ /^(\w+):?(\d+)?$/) {
      # Proxy is in format:
      #   localhost:1337
      #   localhost
      # Add a scheme so LWP can use it.
      # Other formats are passed to LWP directly.
      my ($host, $port) = ($1, $2);

      $port ||= 1080; # socks by default

      $proxy = "socks://$host:$port";

    }
    print STDERR "Using proxy server $proxy\n"
      if $App::get_flash_videos::opt{debug};

    $browser->proxy([qw[http https]] => $proxy);
{'fraction_non_alphanumeric': 0.167276529012023, 'fraction_numerical': 0.01150026136957658, 'mean_word_length': 3.3698630136986303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1332336', 'n_tokens_mistral': 708, 'n_tokens_neox': 663, 'n_words': 182}