<commit_before>package SourcesTest;

use v5.10;
use strict;
use warnings;

use Test::More;
use Text::CSV;
use HTTP::Request;
use LWP::UserAgent;
use URI;
use Carp;

sub new {
    my $class = shift;
    my $file  = shift;

    my $self = {
            input_file => $file,
        };
    bless $self, $class;

    return $self;
}

sub run_tests {
    my $self = shift;

    my $csv = Text::CSV->new({ binary => 1 })
                or die "Cannot use CSV: " . Text::CSV->error_diag();

    open( my $fh, "<", $self->{'input_file'} )
            or die "$self->{'input_file'}: $!";

    my $names = $csv->getline( $fh );
    $csv->column_names( @$names );

    while ( my $row = $csv->getline_hr( $fh ) ) {
        $self->test($row);
    }
}

sub test_source_line {
    my $self = shift;
    my $row  = shift;

    # only test closed sources
    my $mapping_status = lc($row->{'Whole Tag'} // '');
    return if $mapping_status && $mapping_status !~ m{\bclosed\b};

    my $old_url = $row->{'Old Url'};
    my $new_url = $row->{'New Url'};

    ok($old_url ne '#REF!', "Old Url '${old_url}' should not be '#REF!'");
    ok($old_url =~ m{^https?://}, "Old Url '${old_url}' should be a full URL");

    my $old_uri = URI->new($old_url);
    is($old_uri, $old_url, "Old Url '${old_url}' should be a valid URL");

    my $status = $row->{'Status'} // '';
<commit_msg>ditched concept of Whole Tag from validate_csv<commit_after>package SourcesTest;

use v5.10;
use strict;
use warnings;

use Test::More;
use Text::CSV;
use HTTP::Request;
use LWP::UserAgent;
use URI;
use Carp;

sub new {
    my $class = shift;
    my $file  = shift;

    my $self = {
            input_file => $file,
        };
    bless $self, $class;

    return $self;
}

sub run_tests {
    my $self = shift;

    my $csv = Text::CSV->new({ binary => 1 })
                or die "Cannot use CSV: " . Text::CSV->error_diag();

    open( my $fh, "<", $self->{'input_file'} )
            or die "$self->{'input_file'}: $!";

    my $names = $csv->getline( $fh );
    $csv->column_names( @$names );

    while ( my $row = $csv->getline_hr( $fh ) ) {
        $self->test($row);
    }
}

sub test_source_line {
    my $self = shift;
    my $row  = shift;

    my $old_url = $row->{'Old Url'};
    my $new_url = $row->{'New Url'};
    my $status = $row->{'Status'} // '';

    ok($old_url ne '#REF!', "Old Url '${old_url}' should not be '#REF!'");
    ok($old_url =~ m{^https?://}, "Old Url '${old_url}' should be a full URL");

    my $old_uri = URI->new($old_url);
    is($old_uri, $old_url, "Old Url '${old_url}' should be a valid URL");

{'fraction_non_alphanumeric': 0.19622931896883417, 'fraction_numerical': 0.0030781069642170067, 'mean_word_length': 2.6262203626220364, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1156794', 'n_tokens_mistral': 1005, 'n_tokens_neox': 966, 'n_words': 269}