<commit_before># /=====================================================================\ #
# | NNexus Autolinker                                                   | #
# | Indexing Plug-in, nCatLab.org domain                                | #
# |=====================================================================| #
# | Part of the Planetary project: http://trac.mathweb.org/planetary    | #
# |  Research software, produced as part of work done by:               | #
# |  the KWARC group at Jacobs University                               | #
# | Copyright (c) 2012                                                  | #
# | Released under the MIT License (MIT)                                | #
# |---------------------------------------------------------------------| #
# | Adapted from the original NNexus code by                            | #
# |                                  <NAME> and <NAME>     | #
# |---------------------------------------------------------------------| #
# | <NAME> <<EMAIL>>                  #_#     | #
# | http://kwarc.info/people/dginev                            (o o)    | #
# \=========================================================ooo==U==ooo=/ #
package NNexus::Index::Nlab;
use warnings;
use strict;
use base qw(NNexus::Index::Template);

# 1. We want to start from the All Pages entry point, containing the list of all categories
sub domain_root { "http://ncatlab.org/nlab/show/All+Pages"; }
our $nlab_base = 'http://ncatlab.org';
# 2. Candidate links to subcategories and concept pages
our $category_test = qr'^ncatlab\.org/nlab/list/(.+)$';

sub candidate_links {
  my ($self)=@_;
  my $url = $self->current_url;
  # Add links from subcategory pages
  if ($url =~ /$category_test/ ) {
    my $dom = $self->current_dom;
    my @anchors = map {$_->find('a')->each} $dom->find('li[class="page"]')->each;
    my @concept_pages = map {$nlab_base . $_} grep {defined} map {$_->{'href'}} @anchors;
    return \@concept_pages; }
  elsif ($url =~ /All\+Pages$/) {
    # First page, collect all categories:
    my $dom = $self->current_dom;
    my @anchors = $dom->find('ul')->[0]->find('a')->each;
    my @category_pages = map {$nlab_base . $_} grep {defined} map {$_->{'href'}} @anchors;
    return \@category_pages; }
  else {return [];} # skip leaves
}

# Index a concept page, ignore category pages
sub index_page { 
  my ($self) = @_;
  my $url = $self->current_url;
  # Nothing to do in category pages
  return [] if ((! $self->leaf_test($url)) || ($url =~ /All\+Pages$/));
  my $dom = $self->current_dom;
  
  my ($concept) = map {lc($_)} $dom->find('h1')->text->each;
  my $categories = $self->current_categories || ['XX-XX'];

  return [{ url => $url,
	 concept => $concept,
   scheme => 'nlab',
	 categories => $categories,
   }]; }

sub candidate_categories {
	my ($self) = @_;
	if ($self->current_url =~ /$category_test/ ) {
    my $category_name = $1;
    $category_name =~ s/\+/ /g;
		return [$category_name]; }
  else {
		return $self->current_categories; }}

sub leaf_test { $_[1] !~ /$category_test/ }

1;

__END__

=pod

=head1 NAME
<commit_msg> robustness enhancements for some pages that deviate from the overall layout pattern<commit_after># /=====================================================================\ #
# | NNexus Autolinker                                                   | #
# | Indexing Plug-in, nCatLab.org domain                                | #
# |=====================================================================| #
# | Part of the Planetary project: http://trac.mathweb.org/planetary    | #
# |  Research software, produced as part of work done by:               | #
# |  the KWARC group at Jacobs University                               | #
# | Copyright (c) 2012                                                  | #
# | Released under the MIT License (MIT)                                | #
# |---------------------------------------------------------------------| #
# | Adapted from the original NNexus code by                            | #
# |                                  <NAME> and <NAME>     | #
# |---------------------------------------------------------------------| #
# | <NAME> <<EMAIL>>                  #_#     | #
# | http://kwarc.info/people/dginev                            (o o)    | #
# \=========================================================ooo==U==ooo=/ #
package NNexus::Index::Nlab;
use warnings;
use strict;
use base qw(NNexus::Index::Template);

# 1. We want to start from the All Pages entry point, containing the list of all categories
sub domain_root { "http://ncatlab.org/nlab/show/All+Pages"; }
our $nlab_base = 'http://ncatlab.org';
# 2. Candidate links to subcategories and concept pages
our $category_test = qr'^ncatlab\.org/nlab/list/(.+)$';

sub candidate_links {
  my ($self)=@_;
  my $url = $self->current_url;
  # Add links from subcategory pages
  if ($url =~ /$category_test/ ) {
    my $dom = $self->current_dom;
    my @anchors = map {$_->find('a')->each} $dom->find('li[class="page"]')->each;
    my @concept_pages = map {$nlab_base . $_} grep {defined} map {$_->{'href'}} @anchors;
    return \@concept_pages; }
  elsif ($url =~ /All\+Pages$/) {
    # First page, collect all categories:
    my $dom = $self->current_dom;
    my @anchors = $dom->find('ul')->[0]->find('a')->each;
    my @category_pages = map {$nlab_base . $_} grep {defined} map {$_->{'href'}} @anchors;
    return \@category_pages; }
  else {return [];} # skip leaves
}

# Index a concept page, ignore category pages
sub index_page { 
  my ($self) = @_;
  my $url = $self->current_url;
  # Nothing to do in category pages
  return [] if ((! $self->leaf_test($url)) || ($url =~ /All\+Pages$/));
  my $dom = $self->current_dom;
  my $h1 = $dom->find('h1')->[0];
  my $concept = $h1 && lc($h1->text);
  my $categories = $self->current_categories || ['XX-XX'];

  return [{ url => $url,
	 concept => $concept,
   scheme => 'nlab',
	 categories => $categories,
   }]; }

sub candidate_categories {
	my ($self) = @_;
	if ($self->current_url =~ /$category_test/ ) {
    my $category_name = $1;
    $category_name =~ s/\+/ /g;
		return [$category_name]; }
  else {
		return $self->current_categories; }}

sub leaf_test { $_[1] !~ /$category_test/ }

1;

__END__

=pod

=head1 NAME

{'fraction_non_alphanumeric': 0.25926515031016384, 'fraction_numerical': 0.004453634483855575, 'mean_word_length': 2.125248508946322, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3361108', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1846, 'n_words': 478}