<commit_before>package PhaidraAPI::Controller::Oai;

# based on https://github.com/LibreCat/Dancer-Plugin-Catmandu-OAI

use strict;
use warnings;
use v5.10;
use base 'Mojolicious::Controller';
use Switch;
use Data::MessagePack;
use MIME::Base64 qw(encode_base64url decode_base64url);
use DateTime;
use DateTime::Format::ISO8601;
use DateTime::Format::Strptime;
use BSON::Types qw(:all);
use Clone qw(clone);
use Mojo::JSON qw(encode_json decode_json);
use Mojo::ByteStream qw(b);
use PhaidraAPI::Model::Oai::Openaire;

my $DEFAULT_LIMIT = 100;

my $VERBS = {
  GetRecord => {
    valid    => {metadataPrefix => 1, identifier => 1},
    required => [qw(metadataPrefix identifier)],
  },
  Identify        => {valid => {}, required => []},
  ListIdentifiers => {
    valid => {
      metadataPrefix  => 1,
      from            => 1,
      until           => 1,
      set             => 1,
      resumptionToken => 1
    },
    required => [qw(metadataPrefix)],
  },
  ListMetadataFormats => {valid => {identifier => 1, resumptionToken => 1}, required => []},
  ListRecords         => {
    valid => {
      metadataPrefix  => 1,
      from            => 1,
      until           => 1,
      set             => 1,
      resumptionToken => 1
    },
    required => [qw(metadataPrefix)],
  },
  ListSets => {valid => {resumptionToken => 1}, required => []},
};

sub _deserialize {
  my ($self, $data) = @_;
  my $mp = Data::MessagePack->new->utf8;
  return $mp->unpack(decode_base64url($data));
}

sub _serialize {
  my ($self, $data) = @_;
  my $mp = Data::MessagePack->new->utf8;
  return encode_base64url($mp->pack($data));
}

sub _get_metadata_dc {
  my ($self, $rec) = @_;
  my @el          = qw(contributor, coverage, creator, date, description, format, identifier, language, publisher, relation, rights, source, subject, title, type);
  my %valuesCheck = map {$_ => {}} @el;
  my @metadata;
  if (exists($rec->{bib_publisher})) {
    for my $v (@{$rec->{bib_publisher}}) {
      $valuesCheck{'publisher'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'publisher';
    $field{values} = $rec->{bib_publisher};
    push @metadata, \%field;
  }
  if (exists($rec->{bib_published})) {
    for my $v (@{$rec->{bib_published}}) {
      $valuesCheck{'date'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'date';
    $field{values} = $rec->{bib_published};
    push @metadata, \%field;
  }
  if (exists($rec->{dcterms_datesubmitted})) {
    for my $v (@{$rec->{dcterms_datesubmitted}}) {
      $valuesCheck{'date'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'date';
    $field{values} = $rec->{dcterms_datesubmitted};
    push @metadata, \%field;
  }
  if (exists($rec->{ispartof})) {
    my @ispartofs;
    for my $v (@{$rec->{ispartof}}) {
      push @ispartofs, 'isPartOf:https://' . $self->config->{phaidra}->{baseurl} . '/' . $v;
    }
    my %field;
    $field{name}   = 'relation';
    $field{values} = \@ispartofs;
    push @metadata, \%field;
  }
  for my $k (keys %{$rec}) {
    if ($k =~ m/^dc_([a-z]+)_?([a-z]+)?$/) {
      my $skip = 0;
      for my $v (@{$rec->{$k}}) {
        if ($valuesCheck{$1}{$v}) {
          $skip = 1;
          last;
        }
      }
      next if $skip;
      my %field;
      $field{name}   = $1;
      $field{values} = $rec->{$k};
      $field{lang}   = $2 if $2;
      push @metadata, \%field;
    }
  }
  return \@metadata;
}
<commit_msg>remove abstracts on OAI for IR<commit_after>package PhaidraAPI::Controller::Oai;

# based on https://github.com/LibreCat/Dancer-Plugin-Catmandu-OAI

use strict;
use warnings;
use v5.10;
use base 'Mojolicious::Controller';
use Switch;
use Data::MessagePack;
use MIME::Base64 qw(encode_base64url decode_base64url);
use DateTime;
use DateTime::Format::ISO8601;
use DateTime::Format::Strptime;
use BSON::Types qw(:all);
use Clone qw(clone);
use Mojo::JSON qw(encode_json decode_json);
use Mojo::ByteStream qw(b);
use PhaidraAPI::Model::Oai::Openaire;

my $DEFAULT_LIMIT = 100;

my $VERBS = {
  GetRecord => {
    valid    => {metadataPrefix => 1, identifier => 1},
    required => [qw(metadataPrefix identifier)],
  },
  Identify        => {valid => {}, required => []},
  ListIdentifiers => {
    valid => {
      metadataPrefix  => 1,
      from            => 1,
      until           => 1,
      set             => 1,
      resumptionToken => 1
    },
    required => [qw(metadataPrefix)],
  },
  ListMetadataFormats => {valid => {identifier => 1, resumptionToken => 1}, required => []},
  ListRecords         => {
    valid => {
      metadataPrefix  => 1,
      from            => 1,
      until           => 1,
      set             => 1,
      resumptionToken => 1
    },
    required => [qw(metadataPrefix)],
  },
  ListSets => {valid => {resumptionToken => 1}, required => []},
};

sub _deserialize {
  my ($self, $data) = @_;
  my $mp = Data::MessagePack->new->utf8;
  return $mp->unpack(decode_base64url($data));
}

sub _serialize {
  my ($self, $data) = @_;
  my $mp = Data::MessagePack->new->utf8;
  return encode_base64url($mp->pack($data));
}

sub _get_metadata_dc {
  my ($self, $rec) = @_;
  my @el          = qw(contributor, coverage, creator, date, description, format, identifier, language, publisher, relation, rights, source, subject, title, type);
  my %valuesCheck = map {$_ => {}} @el;
  my @metadata;
  if (exists($rec->{bib_publisher})) {
    for my $v (@{$rec->{bib_publisher}}) {
      $valuesCheck{'publisher'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'publisher';
    $field{values} = $rec->{bib_publisher};
    push @metadata, \%field;
  }
  if (exists($rec->{bib_published})) {
    for my $v (@{$rec->{bib_published}}) {
      $valuesCheck{'date'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'date';
    $field{values} = $rec->{bib_published};
    push @metadata, \%field;
  }
  if (exists($rec->{dcterms_datesubmitted})) {
    for my $v (@{$rec->{dcterms_datesubmitted}}) {
      $valuesCheck{'date'}{$v} = 1;
    }
    my %field;
    $field{name}   = 'date';
    $field{values} = $rec->{dcterms_datesubmitted};
    push @metadata, \%field;
  }
  if (exists($rec->{ispartof})) {
    my @ispartofs;
    for my $v (@{$rec->{ispartof}}) {
      push @ispartofs, 'isPartOf:https://' . $self->config->{phaidra}->{baseurl} . '/' . $v;
    }
    my %field;
    $field{name}   = 'relation';
    $field{values} = \@ispartofs;
    push @metadata, \%field;
  }
  for my $k (keys %{$rec}) {
    if ($k =~ m/^dc_([a-z]+)_?([a-z]+)?$/) {
      my $skip = 0;
      for my $v (@{$rec->{$k}}) {
        if ($valuesCheck{$1}{$v}) {
          $skip = 1;
          last;
        }
      }
      next if $skip;
      my %field;
      $field{name} = $1;
      if ($1 eq 'description') {
        if (exists($rec->{ispartof})) {
          if ($rec->{ispartof} eq $self->app->config->{ir}->{ircollection}) {
            $field{values} = "The abstract is available here: https://" . $self->app->config->{ir}->{baseurl} . "/" . $rec->{pid};
          }
        }
      }
      else {
        $field{values} = $rec->{$k};
      }
      $field{lang} = $2 if $2;
      push @metadata, \%field;
    }
  }
  return \@metadata;
}

{'fraction_non_alphanumeric': 0.19621848739495798, 'fraction_numerical': 0.013025210084033614, 'mean_word_length': 2.665811088295688, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 126, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2106019', 'n_tokens_mistral': 2662, 'n_tokens_neox': 2566, 'n_words': 543}