<commit_before>package Poller::CPAN;
use strict;
use Poller;
use vars qw(@ISA);
@ISA = qw(Poller);
use Regions qw(%country_code %country_continent);
use LWP::Simple qw(mirror RC_OK RC_NOT_MODIFIED);

sub new {
  warn "loading CPAN poller module ...";
  my $proto = shift;
  my $class = ref($proto) || $proto;
  $class->SUPER::new();
}

sub check_data {
  my $self = shift;
  $self->mirror_url("http://mirror.cpan.org/rrlist.txt");
}

sub convert {
  my $self = shift;
  my $file = $self->data_file;
  open DATA, $file or die "Could not open $file: $!";
  my @lb_data = ();
  while (<DATA>) {
    next unless m/\S/;
    chomp;
    my ($url, $host, $bandwidth, $contact, $country) = map { $_ =~ s/\s*$//; $_ } split /\t/, $_;
    next unless $url =~ m!ftp://[^/]+/pub/CPAN/!;
    #print "$host / $bandwidth / $country\n";
    my $ip = join ".", unpack('C4',((gethostbyname($host))[4])[0]); 
    
    next if $ip eq "172.16.31.10";

    my $country_code = $country_code{$country};<commit_msg>don't exclude mirrors that are registered without the ending / in their ftp url<commit_after>package Poller::CPAN;
use strict;
use Poller;
use vars qw(@ISA);
@ISA = qw(Poller);
use Regions qw(%country_code %country_continent);
use LWP::Simple qw(mirror RC_OK RC_NOT_MODIFIED);

sub new {
  warn "loading CPAN poller module ...";
  my $proto = shift;
  my $class = ref($proto) || $proto;
  $class->SUPER::new();
}

sub check_data {
  my $self = shift;
  $self->mirror_url("http://mirror.cpan.org/rrlist.txt");
}

sub convert {
  my $self = shift;
  my $file = $self->data_file;
  open DATA, $file or die "Could not open $file: $!";
  my @lb_data = ();
  while (<DATA>) {
    next unless m/\S/;
    chomp;
    my ($url, $host, $bandwidth, $contact, $country) = map { $_ =~ s/\s*$//; $_ } split /\t/, $_;
    next unless $url =~ m!ftp://[^/]+/pub/CPAN/?\b!;
    #print "$host / $bandwidth / $country\n";
    my $ip = join ".", unpack('C4',((gethostbyname($host))[4])[0]); 
    
    next if $ip eq "172.16.31.10";

    my $country_code = $country_code{$country};
{'fraction_non_alphanumeric': 0.20127015144113336, 'fraction_numerical': 0.01172447484123107, 'mean_word_length': 3.413793103448276, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1958131', 'n_tokens_mistral': 829, 'n_tokens_neox': 797, 'n_words': 203}