<commit_before>/*
 * regnum2phyx.js: A tool for converting PhyloRegnum database dumps to Phyx files.
 *
 * Synopsis: regnum2phyx.js [JSON file to process] -o [directory to write files to]
 *
 * PhyloRegnum can be accessed at http://app.phyloregnum.org
 */

// Load necessary modules.
const fs = require('fs');
const path = require('path');
const yargs = require('yargs');
const { has, keys } = require('lodash');

// Helper functions.
function convertAuthorsIntoStrings(authors, lastNameFirst = true) {
  // We combine authors as $first_name $middle_name $last_name.
  // We could instead use foaf:firstName and foaf:lastName, but that's probably
  // unnecessary (http://xmlns.com/foaf/spec/#term_firstName).

  if (lastNameFirst) {
    // Use "last first middle".
    return authors.map(author => (
      `${author.last_name || ''} ${author.first_name || ''}${
        ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? ` ${author.middle_name}` : '')
      }`.trim()
    )).filter(name => name !== '');
  }

  // Use "first middle last".
  return authors.map(author => (
    `${author.first_name || ''} ${
      ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? `${author.middle_name} ` : '')
    }${author.last_name || ''}`.trim()
  )).filter(name => name !== '');
}

function convertAuthorsIntoObjects(authors) {
  return authors.map(author => ({
    name: `${author.last_name || ''}, ${author.first_name || ''}${
      ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? ` ${author.middle_name}` : '')
    }`.trim(),
    alternate: [
      `${author.first_name || ''} ${
        ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? `${author.middle_name} ` : '')
      }${author.last_name || ''}`.trim(),
    ],
    firstname: author.first_name || '',
    lastname: author.last_name || '',
  }));
}

function convertCitation(citation) {
  // Convert a citation from its Regnum format into an ontologized form.
  // We rely on BIBO (https://github.com/structureddynamics/Bibliographic-Ontology-BIBO)
  // quite extensively for this.

  if (!citation) return [];
  if (Array.isArray(citation)) {
    return citation.map(c => convertCitation(c)).reduce((acc, val) => acc.concat(val), []);
  }

  // Figure out the citation type.
  const citationTypes = {
    journal: 'bibo:Article',
    book: 'bibo:Book',
    book_section: 'bibo:BookSection',
  };
  const citationType = citationTypes[citation.citation_type || 'journal'];
  if (!citationType) {
    throw new Error(`Unknown citation type: '${citation.citation_type}'`);
  }

  // Do we have a title and a year? If not, skip this entry.
  if (citation.title === undefined || citation.title.trim() === '' || citation.year === undefined || citation.year.trim() === '') {
    return [];
  }

  // No authors, title or year? Ignore.
  if (!citation.authors || !citation.title || !citation.year) return [];

  const entry = {
    type: citation.citation_type || 'journal',
    title: (citation.title || '').trim(),
    year: (citation.year || '').trim(),
    authors: convertAuthorsIntoObjects(citation.authors),
    journal: {
      name: (citation.journal || '').trim(),
      volume: (citation.volume || '').trim(),
      pages: (citation.pages || '').trim(),
      identifier: [
        {
          type: 'isbn',
          id: (citation.isbn || '').trim(),
        },
        {
          type: 'issn',
          id: (citation.isbn || '').trim(),
        },
      ],
    },
    identifier: {
      type: 'doi',
      id: (citation.doi || '').trim(),
    },
  };

  return [entry];
}<commit_msg>Improved citation generation.<commit_after>/*
 * regnum2phyx.js: A tool for converting PhyloRegnum database dumps to Phyx files.
 *
 * Synopsis: regnum2phyx.js [JSON file to process] -o [directory to write files to]
 *
 * PhyloRegnum can be accessed at http://app.phyloregnum.org
 */

// Load necessary modules.
const fs = require('fs');
const path = require('path');
const yargs = require('yargs');
const { has, keys, pickBy } = require('lodash');

// Helper functions.
function convertAuthorsIntoStrings(authors, lastNameFirst = true) {
  // We combine authors as $first_name $middle_name $last_name.
  // We could instead use foaf:firstName and foaf:lastName, but that's probably
  // unnecessary (http://xmlns.com/foaf/spec/#term_firstName).

  if (lastNameFirst) {
    // Use "last first middle".
    return authors.map(author => (
      `${author.last_name || ''} ${author.first_name || ''}${
        ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? ` ${author.middle_name}` : '')
      }`.trim()
    )).filter(name => name !== '');
  }

  // Use "first middle last".
  return authors.map(author => (
    `${author.first_name || ''} ${
      ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? `${author.middle_name} ` : '')
    }${author.last_name || ''}`.trim()
  )).filter(name => name !== '');
}

function convertAuthorsIntoObjects(authors) {
  return authors
    .filter(author => author.last_name)
    .map(author => ({
      name: `${author.last_name || ''}, ${author.first_name || ''}${
        ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? ` ${author.middle_name}` : '')
      }`.trim(),
      alternate: [
        `${author.first_name || ''} ${
          ((has(author, 'middle_name') && author.middle_name.trim() !== '') ? `${author.middle_name} ` : '')
        }${author.last_name || ''}`.trim(),
      ],
      firstname: author.first_name || '',
      lastname: author.last_name || '',
    }));
}

function convertCitation(citation) {
  // Convert a citation from its Regnum format into an ontologized form.
  // We rely on BIBO (https://github.com/structureddynamics/Bibliographic-Ontology-BIBO)
  // quite extensively for this.

  if (!citation) return [];
  if (Array.isArray(citation)) {
    return citation.map(c => convertCitation(c)).reduce((acc, val) => acc.concat(val), []);
  }

  // Figure out the citation type.
  const citationTypes = {
    journal: 'bibo:Article',
    book: 'bibo:Book',
    book_section: 'bibo:BookSection',
  };
  const citationType = citationTypes[citation.citation_type || 'journal'];
  if (!citationType) {
    throw new Error(`Unknown citation type: '${citation.citation_type}'`);
  }

  // Do we have a title and a year? If not, skip this entry.
  if (citation.title === undefined || citation.title.trim() === '' || citation.year === undefined || citation.year.trim() === '') {
    return [];
  }

  // No authors, title or year? Ignore.
  if (!citation.authors || !citation.title || !citation.year) return [];

  const type = citation.citation_type || 'article';
  const identifiers = [];
  if (citation.doi) identifiers.push({ type: 'doi', id: citation.doi });

  const urls = [];
  if (citation.url) urls.push({ url: citation.url });

  // lodash.pickBy will remove empty keys from the object.
  const entry = pickBy({
    type,
    title: (citation.title || '').trim(),
    section_title: (citation.section_title || '').trim(),
    year: (citation.year || '').trim(),
    edition: (citation.edition || '').trim(),
    authors: convertAuthorsIntoObjects(citation.authors),
    editors: convertAuthorsIntoObjects(citation.editors || []),
    series_editors: convertAuthorsIntoObjects(citation.series_editors || []),
    publisher: (citation.publisher || '').trim(),
    city: (citation.city || '').trim(),
    pages: (citation.pages || '').trim(),
    figure: (citation.figure || '').trim(),
    keywords: (citation.keyword || '').trim(),
    identifier: identifiers,
    link: urls,
  });

  if (type === 'book' || type === 'book_section') {
    // We already have all the details included, so don't add anything else.
  } else if (type === 'journal') {
    // Identifying journal identifiers.
    const journalIdentifiers = [];
    if (citation.isbn) {
      journalIdentifiers.push({ type: 'isbn', id: citation.isbn });
    }
    if (citation.issn) {
      journalIdentifiers.push({ type: 'issn', id: citation.issn });
    }

    // In BibJSON (http://okfnlabs.org/bibjson/), the journal name, volume, number
    // and pages should go into the 'journal' object.
    entry.journal = pickBy({
      name: (citation.journal || '').trim(),
      volume: (citation.volume || '').trim(),
      number: (citation.number || '').trim(),
      pages: (citation.pages || '').trim(),
      identifier: journalIdentifiers,
    });
  } else {
    throw new Error(`Unknown citation type: ${type}`);
  }

  return [entry];
}
{'fraction_non_alphanumeric': 0.1901639344262295, 'fraction_numerical': 0.000468384074941452, 'mean_word_length': 3.279058116232465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '784632', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2641, 'n_words': 630}