<commit_before># 
# Paleobiology Data Service version 1.2
# 
# This file defines version 1.2 of the Paleobiology Data Service.
# 
# Author: <NAME> <<EMAIL>>


package PBData;

use PB2::CommonData;
use PB2::ConfigData;
use PB2::IntervalData;
use PB2::TaxonData;
use PB2::CollectionData;
use PB2::OccurrenceData;
use PB2::DiversityData;
use PB2::ReferenceData;
use PB2::PersonData;

{
    # We start by defining a data service instance for version 1.2
    
    our ($ds2) = Web::DataService->new(
	{ name => '1.2',
	  title => 'PBDB Data Service',
	  version => 'b1',
	  features => 'standard',
	  special_params => 'standard,count=rowcount',
	  path_prefix => 'data1.2/',
	  ruleset_prefix => '1.2:',
	  doc_template_dir => 'doc/1.2' });
    
    
    # We then define the vocabularies that will be used to label the data
    # fields returned by this service.
    
    $ds2->define_vocab(
        { name => 'null', disabled => 1 },	 
	{ name => 'pbdb', title => 'PaleobioDB field names',
	  use_field_names => 1 },
	    "The PBDB vocabulary is derived from the underlying field names and values in the database,",
	    "augmented by a few new fields. For the most part any response that uses this vocabulary",
	    "will be directly comparable to downloads from the PBDB Classic interface.",
	    "This vocabulary is the default for L<text format|node:formats/text> responses.",
	{ name => 'com', title => 'Compact field names' },
	    "The Compact vocabulary is a set of 3-character field names designed to minimize",
	    "the size of the response message.  This is the default for L<JSON format|node:formats/json>",
	    "responses. Some of the field values are similarly abbreviated, while others are conveyed",
	    "in their entirety. For details, see the documentation for the individual response fields.",
	{ name => 'dwc', title => 'Darwin Core', disabled => 1 },
	    "The Darwin Core vocabulary follows the L<Darwin Core standard|http://www.tdwg.org/standards/450/>",
	    "set by the L<TDWG|http://www.tdwg.org/>.  This includes both the field names and field values.",
	    "Because the Darwin Core standard is XML-based, it is very strict.  Many",
	    "but not all of the fields can be expressed in this vocabulary; those that",
	    "cannot are unavoidably left out of the response.");
    
    
    # Then the formats in which data can be returned.
    
    $ds2->define_format(
	{ name => 'json', content_type => 'application/json',
	  doc_node => 'formats/json', title => 'JSON',
	  default_vocab => 'com' },
	    "The JSON format is intended primarily to support client applications,",
	    "including the PBDB Navigator.  Response fields are named using compact",
	    "3-character field names.",
	{ name => 'xml', disabled => 1, content_type => 'text/xml', title => 'XML',
	  doc_node => 'formats/xml', disposition => 'attachment',
	  default_vocab => 'dwc' },
	    "The XML format is intended primarily to support data interchange with",
	    "other databases, using the Darwin Core element set.",
	{ name => 'txt', content_type => 'text/plain',
	  doc_node => 'formats/text', title => 'Comma-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	{ name => 'csv', content_type => 'text/csv',
	  disposition => 'attachment',
	  doc_node => 'formats/text', title => 'Comma-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	{ name => 'tsv', content_type => 'text/tab-separated-values', 
	  disposition => 'attachment',
	  doc_node => 'formats/text', title => 'Tab-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	# { name => 'html', content_type => 'text/html', doc_node => 'formats/html', title => 'HTML',
	#   module => 'Template', disabled => 1 },
	#     "The HTML format returns formatted web pages describing the selected",
	#     "object or objects from the database.",
	{ name => 'ris', content_type => 'application/x-research-info-systems',
	  doc_node => 'formats/ris', title => 'RIS', disposition => 'attachment',
	  encode_as_text => 1, default_vocab => '', module => 'RISFormat'},
	    "The L<RIS format|http://en.wikipedia.org/wiki/RIS_(file_format)> is a",
	    "common format for bibliographic references.",
	{ name => 'png', content_type => 'image/png', module => '',
	  default_vocab => '', doc_node => 'formats/png', title => 'PNG' },
	    "The PNG suffix is used with a few URL paths to fetch images stored",
	    "in the database.");
    
    
    # Then define the URL paths that this subservice will accept.  We start with
    # the root of the hierarchy, which sets defaults for all the rest of the nodes.
    
    $ds2->define_node({ path => '/', 
			public_access => 1,
			doc_default_op_template => 'operation.tt',
			allow_format => 'json,csv,tsv,txt',
			allow_vocab => 'pbdb,com',
			default_save_filename => 'pbdb_data',
			title => 'Documentation' });
    
    # If a default_limit value was defined in the configuration file, get that
    # now so that we can use it to derive limits for certain nodes.
    
    my $base_limit = $ds2->node_attr('/', 'default_limit');
    my $taxa_limit = $base_limit ? $base_limit * 5 : undef;
    $taxa_limit = 20000 if defined $taxa_limit && $taxa_limit < 20000;
    my $ref_limit = $base_limit ? $base_limit * 5 : undef;
    $ref_limit = 10000 if defined $ref_limit && $ref_limit < 10000;
    
    # Configuration. This path is used by clients who need to configure themselves
    # based on parameters supplied by the data service.
    
    $ds2->define_node({ path => 'config',
			place => 10,
			title => 'Client configuration',
			usage => [ "config.json?show=all",
				   "config.txt?show=clusters" ],
			role => 'PB2::ConfigData',
			method => 'get',
			optional_output => '1.2:config:get_map' },
	"This class provides information about the structure, encoding and organization",
	"of the information in the database. It is designed to enable the easy",
	"configuration of client applications.");

    # Occurrences.  These paths are used to fetch information about fossil
    # occurrences known to the database.
    
    $ds2->define_node({ path => 'occs',
			place => 1,
			title => 'Fossil occurrences',
			role => 'PB2::OccurrenceData',
			allow_format => '+xml' },
	"A fossil occurence represents the occurrence of a particular organism at a particular",
	"location in time and space. Each occurrence is a member of a single fossil collection,",
	"and has a taxonomic identification which may be more or less specific.");
    
    $ds2->define_node({ path => 'occs/single',
			place => 1,
			usage => [ "/occs/single.json?id=1001&show=loc", 
				   "/occs/single.txt?id=1001&show=loc,crmod" ],
			method => 'get',
			post_configure_hook => 'prune_field_list',
			output => '1.2:occs:basic',
			optional_output => '1.2:occs:basic_map',
			title => 'Single fossil occurrence'},
	"This path returns information about a single occurrence, selected by its identifier.",
	"Depending upon which output blocks you select, the response will contain some",
	"fields describing the occurrence and some describing the collection to which it belongs.");
    
    $ds2->define_node({ path => 'occs/list',
			place => 2,
			usage => [ "/occs/list.txt?base_name=Cetacea&interval=Miocene&show=loc,phylo",
				   "/occs/list.json?base_name=Cetacea&interval=Miocene&show=loc,phylo" ],
			method => 'list',
			post_configure_hook => 'prune_field_list',
			output => '1.2:occs:basic',
			optional_output => '1.2:occs:basic_map',
			title => 'Lists of fossil occurrences' },
	"This operation returns information about multiple occurrences, selected according to the parameters you provide.",
	"Depending upon which output blocks you select, the response will contain some",
	"fields describing the occurrences and some describing the collections to which they belong.");
    
    $ds2->define_node({ path => 'occs/diversity',
			place => 5,
			usage => [ "/occs/diversity.txt?base_name=Dinosauria^Aves&continent=NOA&count=genera&reso=epoch" ],
			method => 'diversity',
			output => '1.2:occs:diversity',
			default_limit => 1000,
			summary => '1.2:occs:diversity:summary',
			title => 'Fossil diversity over time (full computation)' },
	"This operation returns a tabulation of fossil diversity over time, based on a selected set of occurrences.",
	"You can select the set of occurrences to be analyzed using any of the parameters that are",
	"valid for the L<occs/list|node:occs/list> operation.  This operation can take up a lot of server time,",
	"so if you just want to display a quick overview plot please use the L<occs/quickdev|node:occs/quickdev>",
	"operation.");
    
    $ds2->extended_doc({ path => 'occs/diversity' },
	"It is very important to note that the diversity statistics returned by this",
	"operation reflect only I<the fossil occurrences recorded in this database>, and",
	"not the entire fossil record.  Note also that any occurrences that",
	"are insufficiently resolved either temporally or taxonomically are ignored.",
	"If you wish to apply a different procedure for determining how to count such",
	"occurrences, we suggest that you use the L<occs/list|node:occs/list> operation",
	"instead and apply your procedure directly to the returned list of occurrences.",<commit_msg>Doc fix: bad example link had 'phylo' where it should have had 'class'.<commit_after># 
# Paleobiology Data Service version 1.2
# 
# This file defines version 1.2 of the Paleobiology Data Service.
# 
# Author: <NAME> <<EMAIL>>


package PBData;

use PB2::CommonData;
use PB2::ConfigData;
use PB2::IntervalData;
use PB2::TaxonData;
use PB2::CollectionData;
use PB2::OccurrenceData;
use PB2::DiversityData;
use PB2::ReferenceData;
use PB2::PersonData;

{
    # We start by defining a data service instance for version 1.2
    
    our ($ds2) = Web::DataService->new(
	{ name => '1.2',
	  title => 'PBDB Data Service',
	  version => 'b1',
	  features => 'standard',
	  special_params => 'standard,count=rowcount',
	  path_prefix => 'data1.2/',
	  ruleset_prefix => '1.2:',
	  doc_template_dir => 'doc/1.2' });
    
    
    # We then define the vocabularies that will be used to label the data
    # fields returned by this service.
    
    $ds2->define_vocab(
        { name => 'null', disabled => 1 },	 
	{ name => 'pbdb', title => 'PaleobioDB field names',
	  use_field_names => 1 },
	    "The PBDB vocabulary is derived from the underlying field names and values in the database,",
	    "augmented by a few new fields. For the most part any response that uses this vocabulary",
	    "will be directly comparable to downloads from the PBDB Classic interface.",
	    "This vocabulary is the default for L<text format|node:formats/text> responses.",
	{ name => 'com', title => 'Compact field names' },
	    "The Compact vocabulary is a set of 3-character field names designed to minimize",
	    "the size of the response message.  This is the default for L<JSON format|node:formats/json>",
	    "responses. Some of the field values are similarly abbreviated, while others are conveyed",
	    "in their entirety. For details, see the documentation for the individual response fields.",
	{ name => 'dwc', title => 'Darwin Core', disabled => 1 },
	    "The Darwin Core vocabulary follows the L<Darwin Core standard|http://www.tdwg.org/standards/450/>",
	    "set by the L<TDWG|http://www.tdwg.org/>.  This includes both the field names and field values.",
	    "Because the Darwin Core standard is XML-based, it is very strict.  Many",
	    "but not all of the fields can be expressed in this vocabulary; those that",
	    "cannot are unavoidably left out of the response.");
    
    
    # Then the formats in which data can be returned.
    
    $ds2->define_format(
	{ name => 'json', content_type => 'application/json',
	  doc_node => 'formats/json', title => 'JSON',
	  default_vocab => 'com' },
	    "The JSON format is intended primarily to support client applications,",
	    "including the PBDB Navigator.  Response fields are named using compact",
	    "3-character field names.",
	{ name => 'xml', disabled => 1, content_type => 'text/xml', title => 'XML',
	  doc_node => 'formats/xml', disposition => 'attachment',
	  default_vocab => 'dwc' },
	    "The XML format is intended primarily to support data interchange with",
	    "other databases, using the Darwin Core element set.",
	{ name => 'txt', content_type => 'text/plain',
	  doc_node => 'formats/text', title => 'Comma-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	{ name => 'csv', content_type => 'text/csv',
	  disposition => 'attachment',
	  doc_node => 'formats/text', title => 'Comma-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	{ name => 'tsv', content_type => 'text/tab-separated-values', 
	  disposition => 'attachment',
	  doc_node => 'formats/text', title => 'Tab-separated text',
	  default_vocab => 'pbdb' },
	    "The text formats (txt, tsv, csv) are intended primarily for researchers",
	    "downloading data from the database.  These downloads can easily be",
	    "loaded into spreadsheets or other analysis tools.  The field names are",
	    "taken from the PBDB Classic interface, for compatibility with existing",
	    "tools and analytical procedures.",
	# { name => 'html', content_type => 'text/html', doc_node => 'formats/html', title => 'HTML',
	#   module => 'Template', disabled => 1 },
	#     "The HTML format returns formatted web pages describing the selected",
	#     "object or objects from the database.",
	{ name => 'ris', content_type => 'application/x-research-info-systems',
	  doc_node => 'formats/ris', title => 'RIS', disposition => 'attachment',
	  encode_as_text => 1, default_vocab => '', module => 'RISFormat'},
	    "The L<RIS format|http://en.wikipedia.org/wiki/RIS_(file_format)> is a",
	    "common format for bibliographic references.",
	{ name => 'png', content_type => 'image/png', module => '',
	  default_vocab => '', doc_node => 'formats/png', title => 'PNG' },
	    "The PNG suffix is used with a few URL paths to fetch images stored",
	    "in the database.");
    
    
    # Then define the URL paths that this subservice will accept.  We start with
    # the root of the hierarchy, which sets defaults for all the rest of the nodes.
    
    $ds2->define_node({ path => '/', 
			public_access => 1,
			doc_default_op_template => 'operation.tt',
			allow_format => 'json,csv,tsv,txt',
			allow_vocab => 'pbdb,com',
			default_save_filename => 'pbdb_data',
			title => 'Documentation' });
    
    # If a default_limit value was defined in the configuration file, get that
    # now so that we can use it to derive limits for certain nodes.
    
    my $base_limit = $ds2->node_attr('/', 'default_limit');
    my $taxa_limit = $base_limit ? $base_limit * 5 : undef;
    $taxa_limit = 20000 if defined $taxa_limit && $taxa_limit < 20000;
    my $ref_limit = $base_limit ? $base_limit * 5 : undef;
    $ref_limit = 10000 if defined $ref_limit && $ref_limit < 10000;
    
    # Configuration. This path is used by clients who need to configure themselves
    # based on parameters supplied by the data service.
    
    $ds2->define_node({ path => 'config',
			place => 10,
			title => 'Client configuration',
			usage => [ "config.json?show=all",
				   "config.txt?show=clusters" ],
			role => 'PB2::ConfigData',
			method => 'get',
			optional_output => '1.2:config:get_map' },
	"This class provides information about the structure, encoding and organization",
	"of the information in the database. It is designed to enable the easy",
	"configuration of client applications.");

    # Occurrences.  These paths are used to fetch information about fossil
    # occurrences known to the database.
    
    $ds2->define_node({ path => 'occs',
			place => 1,
			title => 'Fossil occurrences',
			role => 'PB2::OccurrenceData',
			allow_format => '+xml' },
	"A fossil occurence represents the occurrence of a particular organism at a particular",
	"location in time and space. Each occurrence is a member of a single fossil collection,",
	"and has a taxonomic identification which may be more or less specific.");
    
    $ds2->define_node({ path => 'occs/single',
			place => 1,
			usage => [ "/occs/single.json?id=1001&show=loc", 
				   "/occs/single.txt?id=1001&show=loc,crmod" ],
			method => 'get',
			post_configure_hook => 'prune_field_list',
			output => '1.2:occs:basic',
			optional_output => '1.2:occs:basic_map',
			title => 'Single fossil occurrence'},
	"This path returns information about a single occurrence, selected by its identifier.",
	"Depending upon which output blocks you select, the response will contain some",
	"fields describing the occurrence and some describing the collection to which it belongs.");
    
    $ds2->define_node({ path => 'occs/list',
			place => 2,
			usage => [ "/occs/list.txt?base_name=Cetacea&interval=Miocene&show=loc,class",
				   "/occs/list.json?base_name=Cetacea&interval=Miocene&show=loc,class" ],
			method => 'list',
			post_configure_hook => 'prune_field_list',
			output => '1.2:occs:basic',
			optional_output => '1.2:occs:basic_map',
			title => 'Lists of fossil occurrences' },
	"This operation returns information about multiple occurrences, selected according to the parameters you provide.",
	"Depending upon which output blocks you select, the response will contain some",
	"fields describing the occurrences and some describing the collections to which they belong.");
    
    $ds2->define_node({ path => 'occs/diversity',
			place => 5,
			usage => [ "/occs/diversity.txt?base_name=Dinosauria^Aves&continent=NOA&count=genera&reso=epoch" ],
			method => 'diversity',
			output => '1.2:occs:diversity',
			default_limit => 1000,
			summary => '1.2:occs:diversity:summary',
			title => 'Fossil diversity over time (full computation)' },
	"This operation returns a tabulation of fossil diversity over time, based on a selected set of occurrences.",
	"You can select the set of occurrences to be analyzed using any of the parameters that are",
	"valid for the L<occs/list|node:occs/list> operation.  This operation can take up a lot of server time,",
	"so if you just want to display a quick overview plot please use the L<occs/quickdev|node:occs/quickdev>",
	"operation.");
    
    $ds2->extended_doc({ path => 'occs/diversity' },
	"It is very important to note that the diversity statistics returned by this",
	"operation reflect only I<the fossil occurrences recorded in this database>, and",
	"not the entire fossil record.  Note also that any occurrences that",
	"are insufficiently resolved either temporally or taxonomically are ignored.",
	"If you wish to apply a different procedure for determining how to count such",
	"occurrences, we suggest that you use the L<occs/list|node:occs/list> operation",
	"instead and apply your procedure directly to the returned list of occurrences.",
{'fraction_non_alphanumeric': 0.11844924995129553, 'fraction_numerical': 0.010033119033703488, 'mean_word_length': 3.7884794776119404, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 271, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7437715', 'n_tokens_mistral': 6535, 'n_tokens_neox': 5914, 'n_words': 2251}