<commit_before>package RSGet::Plugin;
# This file is an integral part of rsget.pl downloader.
#
# 2009 (c) <NAME> <<EMAIL>>
#		This program is free software,
# you may distribute it under GPL v2 or newer.

use strict;
use warnings;
use RSGet::Processor;
use RSGet::Tools;
set_rev qq$Id$;

my %getters;

sub read_file($)
{
	my $self = shift;
	my $file = $self->{file};

	open F_IN, '<', $file or return;

	my %opts = (
		uri => [],
		map { $_ => undef } qw(name short slots cookie status),
	);
	my $opts = join "|", keys %opts;

	my %parts = (
		map { $_ => [] } qw(unify pre start perl),
	);
	my $parts = join "|", keys %parts;

	my $part;
	while ( <F_IN> ) {
		chomp;
		next if /^\s*#/;
		next if /^\s*$/;

		if ( /^($parts)\s*:/ ) {
			$part = $1;
			last;
		}

		my ( $key, $value );
		unless ( ($key, $value) = /^($opts)\s*:\s+(.*)$/ ) {
			warn "$file: unrecognized line: $_\n";
			next;
		}

		if ( ref $opts{ $key } ) {
			push @{ $opts{ $key } }, $value;
		} else {
			warn "$file: $key overwritten (changed from '$opts{ $key }' to '$value')\n"
				if defined $opts{ $key };
			$opts{ $key } = $value;
		}
	}

	while ( <F_IN> ) {
		chomp;
		next if /^\s*#/;
		next if /^\s*$/;

		if ( /^($parts)\s*:/ ) {
			$part = $1;
			if ( $part eq "perl" ) {
				my @perl = <F_IN>;
				$parts{perl} = \@perl;
			}
			next;
		}

		push @{ $parts{ $part } }, $_;
	}

	close F_IN;

	foreach my $k ( keys %opts ) {
		$self->{ $k } = $opts{ $k };
	}

	return \%parts;
}

sub check_opts
{
	my $self = shift;
	my $file = shift;
	my $plugin_class = shift;

	unless ( @{$self->{uri}} ) {
		return "Can't find 'uri:'\n";
	}

	foreach ( qw(name short) ) {
		next if $self->{$_};
		return "Can't find '$_:'\n";
	}

	$file =~ m{.*/(.*?)$};
	my $fname = $1;
	if ( $fname eq $self->{name} ) {
		$self->{pkg} = $plugin_class."::". $self->{name};
	} else {
		return "Name field: '$self->{name}' differs from file name\n";
	}

	if ( $self->{status} and $self->{status} =~ /^OK(\s+.*)?$/ ) {
		return "";
	}

	return "Incorrect status\n";
}

sub check_parts
{
	my $class = shift;
	my $parts = shift;

	unless ( @{ $parts->{start} } ) {
		return "Can't find start\n";
	}

	return "";
}

sub eval_uris
{
	my $self = shift;
	my $in = $self->{uri};
	my @out;

	local $SIG{__DIE__};
	delete $SIG{__DIE__};

	foreach my $uri_text ( @$in ) {
		my $re = eval $uri_text;
		if ( $@ ) {
			warn "Problem with uri $uri_text: $@\n";
		} elsif ( not $re ) {
			warn "Problem with uri $uri_text\n";
		} elsif ( not ref $re or ref $re ne "Regexp" ) {
			warn "URI $uri_text is not a regular expression\n";
		} else {
			push @out, $re;
		}
	}

	$self->{urire} = \@out;
}

sub new
{
	my $class = shift;
	my $type = shift;
	my $file = shift;

	my $self = {
		file => $file,
		class => $type,
	};
	bless $self, $class;

	my $parts = $self->read_file();
	return undef unless $parts;
	my $error = "";
	$error .= $self->check_opts( $file, $type );
	$error .= $self->check_parts( $parts );

	$self->eval_uris();
	return undef unless @{ $self->{urire} };

	$self->{error} = "$self->{pkg} plugin error: $error" if $error;
	p $file . ": " . $self->{error} if $error;

	$self->compile if setting( "debug" );

	return $self;
}

sub compile
{
	my $self = shift;
	$self->{compiled} = 1;
	return if $self->{error};
	p "Compiling $self->{pkg} plugin";

	my $parts = $self->read_file();
	unless ( $parts ) {
		$self->{error} = "$self->{pkg} compilation error: cannot read file $self->{file}";
		p "Compilation failed";
	}

	my $unify = RSGet::Processor::compile( $self, $parts );

	if ( ref $unify and ref $unify eq "CODE" ) {
		$self->{unify} = $unify;
		p "Compilation successful";
	} else {
		$self->{error} = "$self->{pkg} compilation error";
		p "Compilation failed";
	}
}

sub can_do
{
	my $self = shift;
	my $uri = shift;

	foreach my $re ( @{ $self->{urire} } ) {
		return 1 if $uri =~ m{^http://(?:www\.)?$re};
	}
	return 0;
}
<commit_msg>- added main web and tos information<commit_after>package RSGet::Plugin;
# This file is an integral part of rsget.pl downloader.
#
# 2009 (c) <NAME> <<EMAIL>>
#		This program is free software,
# you may distribute it under GPL v2 or newer.

use strict;
use warnings;
use RSGet::Processor;
use RSGet::Tools;
set_rev qq$Id$;

my %getters;

sub read_file($)
{
	my $self = shift;
	my $file = $self->{file};

	open F_IN, '<', $file or return;

	my %opts = (
		uri => [],
		map { $_ => undef } qw(name short slots cookie status web tos),
	);
	my $opts = join "|", keys %opts;

	my %parts = (
		map { $_ => [] } qw(unify pre start perl),
	);
	my $parts = join "|", keys %parts;

	my $part;
	while ( <F_IN> ) {
		chomp;
		next if /^\s*#/;
		next if /^\s*$/;

		if ( /^($parts)\s*:/ ) {
			$part = $1;
			last;
		}

		my ( $key, $value );
		unless ( ($key, $value) = /^($opts)\s*:\s+(.*)$/ ) {
			warn "$file: unrecognized line: $_\n";
			next;
		}

		if ( ref $opts{ $key } ) {
			push @{ $opts{ $key } }, $value;
		} else {
			warn "$file: $key overwritten (changed from '$opts{ $key }' to '$value')\n"
				if defined $opts{ $key };
			$opts{ $key } = $value;
		}
	}

	while ( <F_IN> ) {
		chomp;
		next if /^\s*#/;
		next if /^\s*$/;

		if ( /^($parts)\s*:/ ) {
			$part = $1;
			if ( $part eq "perl" ) {
				my @perl = <F_IN>;
				$parts{perl} = \@perl;
			}
			next;
		}

		push @{ $parts{ $part } }, $_;
	}

	close F_IN;

	foreach my $k ( keys %opts ) {
		$self->{ $k } = $opts{ $k };
	}

	return \%parts;
}

sub check_opts
{
	my $self = shift;
	my $file = shift;
	my $plugin_class = shift;

	unless ( @{$self->{uri}} ) {
		return "Can't find 'uri:'\n";
	}

	foreach ( qw(name short) ) {
		next if $self->{$_};
		return "Can't find '$_:'\n";
	}

	$file =~ m{.*/(.*?)$};
	my $fname = $1;
	if ( $fname eq $self->{name} ) {
		$self->{pkg} = $plugin_class."::". $self->{name};
	} else {
		return "Name field: '$self->{name}' differs from file name\n";
	}

	if ( $self->{status} and $self->{status} =~ /^OK(\s+.*)?$/ ) {
		return "";
	}

	return "Incorrect status\n";
}

sub check_parts
{
	my $class = shift;
	my $parts = shift;

	unless ( @{ $parts->{start} } ) {
		return "Can't find start\n";
	}

	return "";
}

sub eval_uris
{
	my $self = shift;
	my $in = $self->{uri};
	my @out;

	local $SIG{__DIE__};
	delete $SIG{__DIE__};

	foreach my $uri_text ( @$in ) {
		my $re = eval $uri_text;
		if ( $@ ) {
			warn "Problem with uri $uri_text: $@\n";
		} elsif ( not $re ) {
			warn "Problem with uri $uri_text\n";
		} elsif ( not ref $re or ref $re ne "Regexp" ) {
			warn "URI $uri_text is not a regular expression\n";
		} else {
			push @out, $re;
		}
	}

	$self->{urire} = \@out;
}

sub new
{
	my $class = shift;
	my $type = shift;
	my $file = shift;

	my $self = {
		file => $file,
		class => $type,
	};
	bless $self, $class;

	my $parts = $self->read_file();
	return undef unless $parts;
	my $error = "";
	$error .= $self->check_opts( $file, $type );
	$error .= $self->check_parts( $parts );

	$self->eval_uris();
	return undef unless @{ $self->{urire} };

	$self->{error} = "$self->{pkg} plugin error: $error" if $error;
	p $file . ": " . $self->{error} if $error;

	$self->compile if setting( "debug" );

	return $self;
}

sub compile
{
	my $self = shift;
	$self->{compiled} = 1;
	return if $self->{error};
	p "Compiling $self->{pkg} plugin ($self->{web})";
	p "$self->{pkg}: make sure you agree with $self->{tos}" if $self->{tos};

	my $parts = $self->read_file();
	unless ( $parts ) {
		$self->{error} = "$self->{pkg} compilation error: cannot read file $self->{file}";
		p "Compilation failed";
	}

	my $unify = RSGet::Processor::compile( $self, $parts );

	if ( ref $unify and ref $unify eq "CODE" ) {
		$self->{unify} = $unify;
		p "Compilation successful";
	} else {
		$self->{error} = "$self->{pkg} compilation error";
		p "Compilation failed";
	}
}

sub can_do
{
	my $self = shift;
	my $uri = shift;

	foreach my $re ( @{ $self->{urire} } ) {
		return 1 if $uri =~ m{^http://(?:www\.)?$re};
	}
	return 0;
}

{'fraction_non_alphanumeric': 0.22198330381988363, 'fraction_numerical': 0.0027826966860612194, 'mean_word_length': 2.8950738916256156, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1302691', 'n_tokens_mistral': 3600, 'n_tokens_neox': 3294, 'n_words': 903}