<commit_before>#!perl
#This program is free software. You may redistribute it under the terms of the Artistic License 2.0.
package Core::Cartooniverse;

use 5.012;
use warnings;
use lib "..";

our $VERSION = v1;

use parent qw(Core::Template);

my $l = Log->new();

#creates the list of known manga
sub _create_list {
	my ($pkg) = @_;
	my %clist;
	$l->trace('create list of known collections');
	my $tree = $pkg->_get_tree('http://www.cartooniverse.co.uk/beta/list.php') or return undef;
	foreach my $content ($tree->look_down('_tag' => 'div', 'class' => 'postcontent')) {
		foreach my $td ($content->look_down('_tag' => 'td')) {
			my $a = $td->look_down(_tag=>'a');
			my $name = HTML::Entities::encode($a->as_trimmed_text(extra_chars => '\xA0'));
			my $href = $a->attr('href');
			my ($id) = ($href =~ m'/([^/]*)/$'i);
			unless ($id) {
				$l->debug("could not parse $href");
				next;
			}
			$id =~ s/\W/_/g;
			$id = 'Cartooniverse_' . $id;
			$clist{$id} = {urlstart => $href. '1/0/', name => $name, urlinfo=>$href};
		}
	}
	$tree->delete();
	return \%clist;
}


#returns a list of keys to search for
sub _searchkeys {
	qw(name);
}

#fetches more information about the comic
sub fetch_info {
	my ($s) = @_;
	return undef if $s->clist()->{moreinfo};
	$l->trace('fetching more info for ', $s->{id});
	my $url = $s->clist()->{urlinfo};
	my $tree = $s->_get_tree($url) or return undef;;
	my $td = $tree->look_down('_tag' => 'div', class=>'postcontent')->look_down(_tag=>'table',align=>'center')->look_down(_tag=>'td'); #first postcontent, first td
	my @p = $td->look_down(_tag=>'p');
	$s->clist()->{Author} = HTML::Entities::encode(($p[1]->content_list())[1]);
	$s->clist()->{Author} =~  s/^\s*:\s*//;
	$s->clist()->{Artist} = HTML::Entities::encode(($p[2]->content_list())[1]);
	$s->clist()->{Artist} =~  s/^\s*:\s*+//;
	$s->clist()->{Scanlator} = HTML::Entities::encode(($p[3]->content_list())[1]);
	$s->clist()->{Scanlator} =~  s/^\s*:\s*+//;
	$s->clist()->{Tags} = join ", ", map {$_->as_trimmed_text()} $p[4]->look_down(class => 'series-info');
	$s->clist()->{Details} = HTML::Entities::encode(($p[6]->content_list())[2]);
	$s->clist()->{moreinfo} = 1;
	return $s->save_clist();
}


package Core::Cartooniverse::Spot;

use parent -norequire, 'Core::Template::Spot';

#$tree
#parses the page to mount it
sub _mount_parse {
	my ($s,$tree) = @_;
	my $img = $tree->look_down(_tag => 'img', id=>'supermanga');
	map {$s->{$_} = $img->attr($_)} qw( src );
	my $js = $tree->look_down(_tag => 'input', value=>'next')->attr('onclick');
	if ($js =~ m#javascript:window.location='(.*)';$#) {
		$s->{next} = $1;
	}
	my $title = $tree->look_down(_tag=>'title')->as_trimmed_text();
	if ($title =~ m/Chapter (\d+) page \d+ : Cartooniverse.co.uk/) { 
		$s->{chapter} = $1;
	}
	($s->{filename}) = ($s->{src} =~ m'/([^/]+)$');
	return 1;
}

1;
<commit_msg>added search parameters to Cartooniverse<commit_after>#!perl
#This program is free software. You may redistribute it under the terms of the Artistic License 2.0.
package Core::Cartooniverse;

use 5.012;
use warnings;
use lib "..";

our $VERSION = v1;

use parent qw(Core::Template);

my $l = Log->new();

#creates the list of known manga
sub _create_list {
	my ($pkg) = @_;
	my %clist;
	$l->trace('create list of known collections');
	my $tree = $pkg->_get_tree('http://www.cartooniverse.co.uk/beta/list.php') or return undef;
	foreach my $content ($tree->look_down('_tag' => 'div', 'class' => 'postcontent')) {
		foreach my $td ($content->look_down('_tag' => 'td')) {
			my $a = $td->look_down(_tag=>'a');
			my $name = HTML::Entities::encode($a->as_trimmed_text(extra_chars => '\xA0'));
			my $href = $a->attr('href');
			my ($id) = ($href =~ m'/([^/]*)/$'i);
			unless ($id) {
				$l->debug("could not parse $href");
				next;
			}
			$id =~ s/\W/_/g;
			$id = 'Cartooniverse_' . $id;
			$clist{$id} = {urlstart => $href. '1/0/', name => $name, urlinfo=>$href};
		}
	}
	$tree->delete();
	return \%clist;
}


#returns a list of keys to search for
sub _searchkeys {
	qw(name Author Artist Scanlator Tags);
}

#fetches more information about the comic
sub fetch_info {
	my ($s) = @_;
	return undef if $s->clist()->{moreinfo};
	$l->trace('fetching more info for ', $s->{id});
	my $url = $s->clist()->{urlinfo};
	my $tree = $s->_get_tree($url) or return undef;;
	my $td = $tree->look_down('_tag' => 'div', class=>'postcontent')->look_down(_tag=>'table',align=>'center')->look_down(_tag=>'td'); #first postcontent, first td
	my @p = $td->look_down(_tag=>'p');
	$s->clist()->{Author} = HTML::Entities::encode(($p[1]->content_list())[1]);
	$s->clist()->{Author} =~  s/^\s*:\s*//;
	$s->clist()->{Artist} = HTML::Entities::encode(($p[2]->content_list())[1]);
	$s->clist()->{Artist} =~  s/^\s*:\s*+//;
	$s->clist()->{Scanlator} = HTML::Entities::encode(($p[3]->content_list())[1]);
	$s->clist()->{Scanlator} =~  s/^\s*:\s*+//;
	$s->clist()->{Tags} = join ", ", map {$_->as_trimmed_text()} $p[4]->look_down(class => 'series-info');
	$s->clist()->{Details} = HTML::Entities::encode(($p[6]->content_list())[2]);
	$s->clist()->{moreinfo} = 1;
	return $s->save_clist();
}


package Core::Cartooniverse::Spot;

use parent -norequire, 'Core::Template::Spot';

#$tree
#parses the page to mount it
sub _mount_parse {
	my ($s,$tree) = @_;
	my $img = $tree->look_down(_tag => 'img', id=>'supermanga');
	map {$s->{$_} = $img->attr($_)} qw( src );
	my $js = $tree->look_down(_tag => 'input', value=>'next')->attr('onclick');
	if ($js =~ m#javascript:window.location='(.*)';$#) {
		$s->{next} = $1;
	}
	my $title = $tree->look_down(_tag=>'title')->as_trimmed_text();
	if ($title =~ m/Chapter (\d+) page \d+ : Cartooniverse.co.uk/) { 
		$s->{chapter} = $1;
	}
	($s->{filename}) = ($s->{src} =~ m'/([^/]+)$');
	return 1;
}

1;

{'fraction_non_alphanumeric': 0.2667826086956522, 'fraction_numerical': 0.008347826086956521, 'mean_word_length': 4.492836676217765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 157, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611649', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2319, 'n_words': 480}