<commit_before>/*

Copyright (c) 2000-2021, Board of Trustees of Leland Stanford Jr. University
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

package org.lockss.plugin.atypon;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.lockss.config.TdbAu;
import org.lockss.daemon.ConfigParamDescr;
import org.lockss.extractor.ArticleMetadata;
import org.lockss.extractor.MetadataField;
import org.lockss.plugin.ArchivalUnit;
import org.lockss.plugin.CachedUrl;
import org.lockss.util.UrlUtil;
import org.lockss.util.Logger;
import org.lockss.util.TypedEntryMap;

/**
 * This class contains useful routines to allow a metadata extractor to verify
 * the contents of an ArticleMetadata against the current AU's TDB information.
 * This might be used to avoid emitting an article that is likely due to an
 * overcrawl - currently used by BaseAtypon
 * @author alexohlson
 *
 */
public class BaseAtyponMetadataUtil {
  private static final Logger log = Logger.getLogger(BaseAtyponMetadataUtil.class);

  private static final String BOOK_EISBN_PARAM = "book_eisbn";
  private static final String AU_TYPE = "type";
  private static final Pattern PROTOCOL_PATTERN = Pattern.compile("^(e)?is(b|s)n:(\\s)*", Pattern.CASE_INSENSITIVE);

  /**
   * html "dc.*" information may not contain publication title and volume
   * ris information usually does.
   * If the publication type doesn't match the article type (book v journal)
   * If the AU lists BOTH issn & eissn and the metadata has one that doesn'tm atch
   * If the volume is available, check against the TDB information
   * If the volume wasn't available but the year was, try that
   * If we're still valid and the publication title is available, check
   *   a normalized version of that against a normalized version of tdb pub title
   * @param au
   * @param am
   * @return true if the metadata matches the TDB information
   */
  public static boolean metadataMatchesTdb(ArchivalUnit au,
                                           ArticleMetadata am) {

    boolean isInAu = true;

    //Initial check - are we of the expected type
    // We're only in this method if the type was NOT book or book_chapter
    // or if the article didn't indicate its type (html)
    TdbAu tdbau = au.getTdbAu();
    String au_type = (tdbau == null) ? null : tdbau.getPublicationType();
    // if the tdb publication type is a book or book chapter, then we don't belong
    if ( au_type != null && (MetadataField.PUBLICATION_TYPE_BOOKSERIES.equals(au_type)
        || MetadataField.PUBLICATION_TYPE_BOOK.equals(au_type)) ) {
      // we probably overcrawled and got a journal or proceedings article
      // while preserving a book
      return false;
    }

    // if we do an ISSN check, then we can bypass checking the title later, which is trickier
    Boolean checkedISSN = false;
    String AU_ISSN = (tdbau == null) ? null : normalize_id(tdbau.getPrintIssn());
    String AU_EISSN = (tdbau == null) ? null : normalize_id(tdbau.getEissn());
    // If the tdb lists both values for issn, then check with our found values
    // If we only list one value then don't use this check - what if the au
    // only had the EISSN and the article only listed the ISSN - false negative
    if ( !(StringUtils.isEmpty(AU_ISSN) || StringUtils.isEmpty(AU_EISSN)) ){
      String foundEISSN = normalize_id(am.get(MetadataField.FIELD_EISSN));
      String foundISSN = normalize_id(am.get(MetadataField.FIELD_ISSN));
      checkedISSN = true;
      // don't go crazy. If the EISSN is there and matches, just move on
      if (foundEISSN != null) {
        if (!(foundEISSN.equals(AU_EISSN) || foundEISSN.equals(AU_ISSN)) ) {
          return false;
        }
      } else if (foundISSN != null) {
        // there wasn't an EISSN, so let's check the ISSN
        if (!(foundISSN.equals(AU_ISSN) || foundISSN.equals(AU_EISSN)) ) {
          return false;
        }
      }
    }

    // Use the journalTitle and volume name from the ArticleMetadata
    String foundJournalTitle = am.get(MetadataField.FIELD_PUBLICATION_TITLE);
    String foundVolume = am.get(MetadataField.FIELD_VOLUME);

    // If we got nothing, just return, we can't validate further
    if (StringUtils.isEmpty(foundVolume) && StringUtils.isEmpty(foundJournalTitle)) {
      return isInAu; //return true, we have no way of knowing
    }

    // Check VOLUME/YEAR
    if (!StringUtils.isEmpty(foundVolume)) {
      // Get the AU's volume name from the AU properties. This must be set
      TypedEntryMap tfProps = au.getProperties();
      String AU_volume = tfProps.getString(ConfigParamDescr.VOLUME_NAME.getKey());

      // Do Volume comparison first, it's simpler
      if (isInAu && !(StringUtils.isEmpty(foundVolume))) {
        isInAu =  ( (AU_volume != null) && (AU_volume.equals(foundVolume)));
        log.debug3("After volume check, isInAu :" + isInAu);
      }
    }

    // BEGIN Mark Allen Specific checks
    // get the AU's publisher and check if it is Mark Allen Group.
    String pubName = tdbau.getAttr("publisher");

    if (isInAu && pubName.equals("<NAME>")) {
      log.debug3("MarkAllen Checks");

      String foundDate = am.get(MetadataField.FIELD_DATE);
      String foundDOI = am.get(MetadataField.FIELD_DOI);

      if (!org.apache.commons.lang.StringUtils.isEmpty(foundDate)) {
        String AU_Year = tdbau.getYear();
        // date can come in many formats, so lets try to deal with them,
        // e.g. 2013/09/28, 2013-09-28, 9/28/2013, "September 28, 2013", "2013, Sep 28"
        // other formats are too tricky, e.g. 20130928 so this format should pass the check as well
        // This date check can likely be instantly used for all Atypon,

        String foundYear = null;

        String[] splitDate = foundDate.split("/|-|, ");
        for (String piece : splitDate) {
          if (piece.length() == 4) {
            foundYear = piece;
          }
        }
        isInAu = (!org.apache.commons.lang.StringUtils.isEmpty(foundYear) && (AU_Year != null) && AU_Year.equals(foundYear));
      }

      if (isInAu && !org.apache.commons.lang.StringUtils.isEmpty(foundDOI)) {
        TypedEntryMap tfProps = au.getProperties();
        String JOURNAL_ID = tfProps.getString(ConfigParamDescr.JOURNAL_ID.getKey());
        // laboriously parse this out to the journal id that is embedded in the DOI
        // DOI for MarkAllen stuff always looks like this:
        // 10.12968/coan.2018.23.1.41
        // this was pulled in but doesnt look like markallen.
        // 10.1111/j.2044-3862.2009.tb00374.x
        // the letters between the '/' and the second '.' is the Journal ID
        // we extract it.
        // To generalize to all of Atypon more analysis should be done for the DOI entries of other publishers
        // as this is prime for false positives.

        int slashIdx = foundDOI.indexOf("/");
        String shouldBeJID = foundDOI.substring(slashIdx + 1, foundDOI.indexOf(".", slashIdx));

        isInAu = ((shouldBeJID != null) && shouldBeJID.equals(JOURNAL_ID));

      }
      log.debug3("foundDate: " + foundDate);
      log.debug3("foundDOI: " + foundDOI);
      if (isInAu) {
        log.debug3("<NAME> Pass");
      }
    }
    // END MARK ALLEN SPECIFIC CHECKS //
<commit_msg>minor refactor of MetadataUtil with Thib<commit_after>/*

Copyright (c) 2000-2021, Board of Trustees of Leland Stanford Jr. University
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

package org.lockss.plugin.atypon;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.lockss.config.TdbAu;
import org.lockss.daemon.ConfigParamDescr;
import org.lockss.extractor.ArticleMetadata;
import org.lockss.extractor.MetadataField;
import org.lockss.plugin.ArchivalUnit;
import org.lockss.plugin.CachedUrl;
import org.lockss.util.UrlUtil;
import org.lockss.util.Logger;
import org.lockss.util.TypedEntryMap;

/**
 * This class contains useful routines to allow a metadata extractor to verify
 * the contents of an ArticleMetadata against the current AU's TDB information.
 * This might be used to avoid emitting an article that is likely due to an
 * overcrawl - currently used by BaseAtypon
 * @author alexohlson
 *
 */
public class BaseAtyponMetadataUtil {
  private static final Logger log = Logger.getLogger(BaseAtyponMetadataUtil.class);

  private static final String BOOK_EISBN_PARAM = "book_eisbn";
  private static final String AU_TYPE = "type";
  private static final Pattern PROTOCOL_PATTERN = Pattern.compile("^(e)?is(b|s)n:(\\s)*", Pattern.CASE_INSENSITIVE);

  /**
   * html "dc.*" information may not contain publication title and volume
   * ris information usually does.
   * If the publication type doesn't match the article type (book v journal)
   * If the AU lists BOTH issn & eissn and the metadata has one that doesn'tm atch
   * If the volume is available, check against the TDB information
   * If the volume wasn't available but the year was, try that
   * If we're still valid and the publication title is available, check
   *   a normalized version of that against a normalized version of tdb pub title
   * @param au
   * @param am
   * @return true if the metadata matches the TDB information
   */
  public static boolean metadataMatchesTdb(ArchivalUnit au,
                                           ArticleMetadata am) {

    boolean isInAu = true;

    //Initial check - are we of the expected type
    // We're only in this method if the type was NOT book or book_chapter
    // or if the article didn't indicate its type (html)
    TdbAu tdbau = au.getTdbAu();
    String au_type = (tdbau == null) ? null : tdbau.getPublicationType();
    // if the tdb publication type is a book or book chapter, then we don't belong
    if ( au_type != null && (MetadataField.PUBLICATION_TYPE_BOOKSERIES.equals(au_type)
        || MetadataField.PUBLICATION_TYPE_BOOK.equals(au_type)) ) {
      // we probably overcrawled and got a journal or proceedings article
      // while preserving a book
      return false;
    }

    // if we do an ISSN check, then we can bypass checking the title later, which is trickier
    Boolean checkedISSN = false;
    String AU_ISSN = (tdbau == null) ? null : normalize_id(tdbau.getPrintIssn());
    String AU_EISSN = (tdbau == null) ? null : normalize_id(tdbau.getEissn());
    // If the tdb lists both values for issn, then check with our found values
    // If we only list one value then don't use this check - what if the au
    // only had the EISSN and the article only listed the ISSN - false negative
    if ( !(StringUtils.isEmpty(AU_ISSN) || StringUtils.isEmpty(AU_EISSN)) ){
      String foundEISSN = normalize_id(am.get(MetadataField.FIELD_EISSN));
      String foundISSN = normalize_id(am.get(MetadataField.FIELD_ISSN));
      checkedISSN = true;
      // don't go crazy. If the EISSN is there and matches, just move on
      if (foundEISSN != null) {
        if (!(foundEISSN.equals(AU_EISSN) || foundEISSN.equals(AU_ISSN)) ) {
          return false;
        }
      } else if (foundISSN != null) {
        // there wasn't an EISSN, so let's check the ISSN
        if (!(foundISSN.equals(AU_ISSN) || foundISSN.equals(AU_EISSN)) ) {
          return false;
        }
      }
    }

    // Use the journalTitle and volume name from the ArticleMetadata
    String foundJournalTitle = am.get(MetadataField.FIELD_PUBLICATION_TITLE);
    String foundVolume = am.get(MetadataField.FIELD_VOLUME);

    // If we got nothing, just return, we can't validate further
    if (StringUtils.isEmpty(foundVolume) && StringUtils.isEmpty(foundJournalTitle)) {
      return isInAu; //return true, we have no way of knowing
    }

    // Check VOLUME/YEAR
    if (!StringUtils.isEmpty(foundVolume)) {
      // Get the AU's volume name from the AU properties. This must be set
      TypedEntryMap tfProps = au.getProperties();
      String AU_volume = tfProps.getString(ConfigParamDescr.VOLUME_NAME.getKey());

      // Do Volume comparison first, it's simpler
      if (isInAu && !(StringUtils.isEmpty(foundVolume))) {
        isInAu =  ( (AU_volume != null) && (AU_volume.equals(foundVolume)));
        log.debug3("After volume check, isInAu :" + isInAu);
      }
    }

    // BEGIN Mark Allen Specific checks
    // get the AU's publisher and check if it is Mark Allen Group.
    String pubName = tdbau.getPublisherName();

    if (isInAu && pubName.equals("<NAME>")) {
      log.debug3("MarkAllen Checks");

      String foundDate = am.get(MetadataField.FIELD_DATE);
      String foundDOI = am.get(MetadataField.FIELD_DOI);

      if (!StringUtils.isEmpty(foundDate)) {
        String AU_Year = tdbau.getYear();
        // date can come in many formats, so lets try to deal with them,
        // e.g. 2013/09/28, 2013-09-28, 9/28/2013, "September 28, 2013", "2013, Sep 28"
        // other formats are too tricky, e.g. 20130928 so this format should pass the check as well
        // This date check can likely be instantly used for all Atypon,

        String foundYear = null;

        String[] splitDate = foundDate.split("/|-|, ");
        for (String piece : splitDate) {
          if (piece.length() == 4) {
            foundYear = piece;
          }
        }
        isInAu = (!StringUtils.isEmpty(foundYear) && (AU_Year != null) && AU_Year.equals(foundYear));
      }

      if (isInAu && !StringUtils.isEmpty(foundDOI)) {
        String JOURNAL_ID = au.getConfiguration().get(ConfigParamDescr.JOURNAL_ID.getKey());
        // laboriously parse this out to the journal id that is embedded in the DOI
        // DOI for MarkAllen stuff always looks like this:
        // 10.12968/coan.2018.23.1.41
        // this was pulled in but doesnt look like markallen.
        // 10.1111/j.2044-3862.2009.tb00374.x
        // the letters between the '/' and the second '.' is the Journal ID
        // we extract it.
        // To generalize to all of Atypon more analysis should be done for the DOI entries of other publishers
        // as this is prime for false positives.

        int slashIdx = foundDOI.indexOf("/");
        String shouldBeJID = foundDOI.substring(slashIdx + 1, foundDOI.indexOf(".", slashIdx));

        isInAu = ((shouldBeJID != null) && shouldBeJID.equals(JOURNAL_ID));

      }
      log.debug3("foundDate: " + foundDate);
      log.debug3("foundDOI: " + foundDOI);
      if (isInAu) {
        log.debug3("<NAME>");
      }
    }
    // END MARK ALLEN SPECIFIC CHECKS //

{'fraction_non_alphanumeric': 0.0879990821477742, 'fraction_numerical': 0.01158788435061955, 'mean_word_length': 3.16957665630232, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7659691', 'n_tokens_mistral': 5342, 'n_tokens_neox': 4920, 'n_words': 1869}