<commit_before>/*
 * $Id: KbartConverter.java,v 1.38 2012-12-18 11:38:31 easyonthemayo Exp $
 */

/*

Copyright (c) 2010-2011 Board of Trustees of Leland Stanford Jr. University,
all rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Except as contained in this notice, the name of Stanford University shall not
be used in advertising or otherwise to promote the sale, use or other dealings
in this Software without prior written authorization from Stanford University.

*/

package org.lockss.exporter.kbart;

import java.util.*;
import java.util.concurrent.*;

import org.apache.commons.collections.comparators.ComparatorChain;

import org.lockss.config.*;
import org.lockss.exporter.biblio.*;
import org.lockss.plugin.ArchivalUnit;
import org.lockss.daemon.AuHealthMetric;
import org.lockss.daemon.AuHealthMetric.HealthUnavailableException;
import org.lockss.util.*;

import static org.lockss.exporter.biblio.BibliographicOrderScorer.SORT_FIELD;
import static org.lockss.exporter.biblio.BibliographicUtil.*;
import static org.lockss.exporter.kbart.KbartTitle.Field.*;

/**
 * This class takes metadata in the internal TDB format, and converts it into
 * a format adhering to the KBART recommendations. It extracts the relevant 
 * metadata from internal structures to provide tuples of data suitable for 
 * KBART records. The KBART format focuses at the level of the 
 * <emph>title</emph>, and so the output is made available as 
 * <code>KbartTitle</code> objects via an ordered <code>List</code>. 
 * <p>
 * Note that it is possible to have several <code>KbartTitle</code>
 * objects in a single Publisher which have the same publication title;
 * for example if there are coverage gaps in the license to a particular 
 * publication, there will be a title representing each distinct licensed 
 * period (see 5.3.1.9).
 * <p>
 * KBART suggests (5.3.1.11) that the metadata file should be supplied in 
 * alphabetical order by <emph>title</emph>. This requires an overview of all 
 * titles before the order can be established, whereas the TDB structure is 
 * naturally set up to allow iteration by publisher, it being a higher element 
 * in the hierarchy.
 * <p>
 * Despite this, the class is currently implemented to convert TDB data only 
 * when requested. This means that if the class or one of its methods is used 
 * multiple times, there will be a duplication of effort. To overcome this, 
 * if a class instance is likely to be reused, a thread-safe lazy loading 
 * approach could be implemented. On the other hand, if instantiation seems 
 * like too much work, the whole class could be made static and each method 
 * could take a TDB argument. 
 * <p>
 * There is some justification for recording summary statistics about the 
 * KbartTitles as they are created, for example, what the longest field value 
 * is, or which fields are universally empty. This can be used to inform layout 
 * in outputs that wish to customise the KBART representation. To do this we 
 * could pass all field-setting calls through an exporter-specific summariser. 
 * <p>
 * Note that if the underlying <code>Tdb</code> objects are changed during 
 * iteration the resulting output is undefined.
 * <p>
 * <emph>Note that the <code>title_id</code> field is now filled with the 
 * results of either getIsbn() or getIssn(), which gives a preferred ISBN/ISSN
 * for linking.</emph>
 *
 * <h3>Note: Iteration</h3>
 * The converter can accept an iterator instead of a list, as it deals with a
 * single title at a time. The ordering of those titles can be established in
 * the client using this class, or in the resultant exporter itself; it is not
 * important here except as it relates to the contract made by this class.
 * <p>
 * All non-range versions of convertTitleToKbartTitles perform sorting and
 * return a list, and these could be adapted to omit sorting and return
 * iterators. Ultimately these methods inform
 * ListHoldings.getKbartTitlesForExport and are used in creating a filter, so
 * if the filter is updated to use iterators, we should use the
 * KbartTitleIterator classes instead of the list-returning methods.
 *
 * @author <NAME>
 */
public class KbartConverter {

  private static Logger log = Logger.getLogger("KbartConverter");

  static final String PREFIX = Configuration.PREFIX + "kbart.";

  /**
   * How many threads to provide for title conversions during KBART reporting.
   * The change will take place when the pool is next used for reporting.
   * To reset the pool to the default size, just remove this parameter or set
   * it to zero.
   */
  public static final String PARAM_CONVERT_TITLES_POOL_SIZE =
      PREFIX + "conversionThreadPoolSize";

  /** The number of threads to have available in the thread pool dedicated
   * to converting titles. By default this is one greater than the available
   * number of processors/cores. It can be adjusted at runtime using the config
   * param PARAM_CONVERT_TITLES_POOL_SIZE. */
  public static final int DEFAULT_CONVERT_TITLES_POOL_SIZE =
      Runtime.getRuntime().availableProcessors() + 1;

  /** 
   * The string used as a substitution parameter in the output. Occurrences of 
   * this string will be replaced in local LOCKSS boxes with the protocol, host 
   * and port of ServeContent.
   */
  public static final String LABEL_PARAM_LOCKSS_RESOLVER = "LOCKSS_RESOLVER";
  
  /** The minimum number that will be considered a date of publication. */
  public static final int MIN_PUB_DATE = 1600;
  /**
   * The maximum number of years after the current year that will be considered
   * a valid date of publication.
   */
  public static final int MAX_FUTURE_PUB_DATE = 10;

  /**
   * The thread pool executor used in completion services in converting titles.
   */
  private static final AdjustableFixedSizeThreadPoolExecutor CONVERT_TITLES_EXECUTOR =
      new AdjustableFixedSizeThreadPoolExecutor(DEFAULT_CONVERT_TITLES_POOL_SIZE);

  /**
   * This method should be used to get the executor to use for converting titles;
   * it first resizes the thread pool if the configuration has changed.
   * @return
   */
  private static final AdjustableFixedSizeThreadPoolExecutor getConvertTitlesExecutor() {
    int newSize = CurrentConfig.getIntParam(PARAM_CONVERT_TITLES_POOL_SIZE,
        DEFAULT_CONVERT_TITLES_POOL_SIZE);
    // Set the new size to default if user specified 0
    if (newSize==0) newSize = DEFAULT_CONVERT_TITLES_POOL_SIZE;
    // Resize the pool if the param differs from the current prescribed size
    if (newSize!=CONVERT_TITLES_EXECUTOR.getFixedPoolSize()) {
      boolean success = CONVERT_TITLES_EXECUTOR.setFixedPoolSize(newSize);
      log.info((success?"Resized":"Failed to resize")+" convert titles thread pool to "+newSize);
    }
    return CONVERT_TITLES_EXECUTOR;
  }

  /**
   * Convert the given collection of TdbTitles into KbartTitles.
   * The number of KbartTitles returned is likely to be different to the number
   * of TdbTitles which was originally supplied, as KbartTitles are grouped
   * based on their coverage period rather than purely by title.
   * <p>
   * The KbartTitles resulting from processing the titles are returned in the
   * same order the titles were submitted.
   * <p>
   * This method submits individual title conversion tasks to a thread pool,
   * and retrieves the results in the same order the tasks were submitted.
   * If an execution thread is interrupted or a task throws an exception, the
   * remaining tasks are cancelled, and the exception is logged, wrapped and
   * rethrown.
   *
   * @param titles a collection of TdbTitles
   * @return a list of KbartTitles
   * @throws ConversionException if one of the threads is interrupted or a task throws an exception
   */
  public static List<KbartTitle> convertTitles(final Collection<TdbTitle> titles)
      throws ConversionException {
    if (titles==null) return Collections.emptyList();<commit_msg>Create thread pool lazily, so loading class doesn't precent JVM exit.<commit_after>/*
 * $Id: KbartConverter.java,v 1.39 2013-01-05 04:43:40 tlipkis Exp $
 */

/*

Copyright (c) 2010-2011 Board of Trustees of Leland Stanford Jr. University,
all rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Except as contained in this notice, the name of Stanford University shall not
be used in advertising or otherwise to promote the sale, use or other dealings
in this Software without prior written authorization from Stanford University.

*/

package org.lockss.exporter.kbart;

import java.util.*;
import java.util.concurrent.*;

import org.apache.commons.collections.comparators.ComparatorChain;

import org.lockss.config.*;
import org.lockss.exporter.biblio.*;
import org.lockss.plugin.ArchivalUnit;
import org.lockss.daemon.AuHealthMetric;
import org.lockss.daemon.AuHealthMetric.HealthUnavailableException;
import org.lockss.util.*;

import static org.lockss.exporter.biblio.BibliographicOrderScorer.SORT_FIELD;
import static org.lockss.exporter.biblio.BibliographicUtil.*;
import static org.lockss.exporter.kbart.KbartTitle.Field.*;

/**
 * This class takes metadata in the internal TDB format, and converts it into
 * a format adhering to the KBART recommendations. It extracts the relevant 
 * metadata from internal structures to provide tuples of data suitable for 
 * KBART records. The KBART format focuses at the level of the 
 * <emph>title</emph>, and so the output is made available as 
 * <code>KbartTitle</code> objects via an ordered <code>List</code>. 
 * <p>
 * Note that it is possible to have several <code>KbartTitle</code>
 * objects in a single Publisher which have the same publication title;
 * for example if there are coverage gaps in the license to a particular 
 * publication, there will be a title representing each distinct licensed 
 * period (see 5.3.1.9).
 * <p>
 * KBART suggests (5.3.1.11) that the metadata file should be supplied in 
 * alphabetical order by <emph>title</emph>. This requires an overview of all 
 * titles before the order can be established, whereas the TDB structure is 
 * naturally set up to allow iteration by publisher, it being a higher element 
 * in the hierarchy.
 * <p>
 * Despite this, the class is currently implemented to convert TDB data only 
 * when requested. This means that if the class or one of its methods is used 
 * multiple times, there will be a duplication of effort. To overcome this, 
 * if a class instance is likely to be reused, a thread-safe lazy loading 
 * approach could be implemented. On the other hand, if instantiation seems 
 * like too much work, the whole class could be made static and each method 
 * could take a TDB argument. 
 * <p>
 * There is some justification for recording summary statistics about the 
 * KbartTitles as they are created, for example, what the longest field value 
 * is, or which fields are universally empty. This can be used to inform layout 
 * in outputs that wish to customise the KBART representation. To do this we 
 * could pass all field-setting calls through an exporter-specific summariser. 
 * <p>
 * Note that if the underlying <code>Tdb</code> objects are changed during 
 * iteration the resulting output is undefined.
 * <p>
 * <emph>Note that the <code>title_id</code> field is now filled with the 
 * results of either getIsbn() or getIssn(), which gives a preferred ISBN/ISSN
 * for linking.</emph>
 *
 * <h3>Note: Iteration</h3>
 * The converter can accept an iterator instead of a list, as it deals with a
 * single title at a time. The ordering of those titles can be established in
 * the client using this class, or in the resultant exporter itself; it is not
 * important here except as it relates to the contract made by this class.
 * <p>
 * All non-range versions of convertTitleToKbartTitles perform sorting and
 * return a list, and these could be adapted to omit sorting and return
 * iterators. Ultimately these methods inform
 * ListHoldings.getKbartTitlesForExport and are used in creating a filter, so
 * if the filter is updated to use iterators, we should use the
 * KbartTitleIterator classes instead of the list-returning methods.
 *
 * @author <NAME>
 */
public class KbartConverter {

  private static Logger log = Logger.getLogger("KbartConverter");

  static final String PREFIX = Configuration.PREFIX + "kbart.";

  /**
   * How many threads to provide for title conversions during KBART reporting.
   * The change will take place when the pool is next used for reporting.
   * To reset the pool to the default size, just remove this parameter or set
   * it to zero.
   */
  public static final String PARAM_CONVERT_TITLES_POOL_SIZE =
      PREFIX + "conversionThreadPoolSize";

  /** The number of threads to have available in the thread pool dedicated
   * to converting titles. By default this is one greater than the available
   * number of processors/cores. It can be adjusted at runtime using the config
   * param PARAM_CONVERT_TITLES_POOL_SIZE. */
  public static final int DEFAULT_CONVERT_TITLES_POOL_SIZE =
      Runtime.getRuntime().availableProcessors() + 1;

  /** 
   * The string used as a substitution parameter in the output. Occurrences of 
   * this string will be replaced in local LOCKSS boxes with the protocol, host 
   * and port of ServeContent.
   */
  public static final String LABEL_PARAM_LOCKSS_RESOLVER = "LOCKSS_RESOLVER";
  
  /** The minimum number that will be considered a date of publication. */
  public static final int MIN_PUB_DATE = 1600;
  /**
   * The maximum number of years after the current year that will be considered
   * a valid date of publication.
   */
  public static final int MAX_FUTURE_PUB_DATE = 10;

  /**
   * The thread pool executor used in completion services in converting titles.
   */
  private static AdjustableFixedSizeThreadPoolExecutor CONVERT_TITLES_EXECUTOR = null;

  private static AdjustableFixedSizeThreadPoolExecutor getThreadPoolExecutor() {
    if (CONVERT_TITLES_EXECUTOR == null) {
      CONVERT_TITLES_EXECUTOR =
	new AdjustableFixedSizeThreadPoolExecutor(DEFAULT_CONVERT_TITLES_POOL_SIZE);
    }
    return CONVERT_TITLES_EXECUTOR;
  }

  /**
   * This method should be used to get the executor to use for converting titles;
   * it first resizes the thread pool if the configuration has changed.
   * @return
   */
  private static final AdjustableFixedSizeThreadPoolExecutor getConvertTitlesExecutor() {
    AdjustableFixedSizeThreadPoolExecutor executor = getThreadPoolExecutor();
    int newSize = CurrentConfig.getIntParam(PARAM_CONVERT_TITLES_POOL_SIZE,
        DEFAULT_CONVERT_TITLES_POOL_SIZE);
    // Set the new size to default if user specified 0
    if (newSize==0) newSize = DEFAULT_CONVERT_TITLES_POOL_SIZE;
    // Resize the pool if the param differs from the current prescribed size
    if (newSize!=executor.getFixedPoolSize()) {
      boolean success = executor.setFixedPoolSize(newSize);
      log.info((success?"Resized":"Failed to resize")+" convert titles thread pool to "+newSize);
    }
    return executor;
  }

  /**
   * Convert the given collection of TdbTitles into KbartTitles.
   * The number of KbartTitles returned is likely to be different to the number
   * of TdbTitles which was originally supplied, as KbartTitles are grouped
   * based on their coverage period rather than purely by title.
   * <p>
   * The KbartTitles resulting from processing the titles are returned in the
   * same order the titles were submitted.
   * <p>
   * This method submits individual title conversion tasks to a thread pool,
   * and retrieves the results in the same order the tasks were submitted.
   * If an execution thread is interrupted or a task throws an exception, the
   * remaining tasks are cancelled, and the exception is logged, wrapped and
   * rethrown.
   *
   * @param titles a collection of TdbTitles
   * @return a list of KbartTitles
   * @throws ConversionException if one of the threads is interrupted or a task throws an exception
   */
  public static List<KbartTitle> convertTitles(final Collection<TdbTitle> titles)
      throws ConversionException {
    if (titles==null) return Collections.emptyList();
{'fraction_non_alphanumeric': 0.05982197158179908, 'fraction_numerical': 0.004975949577044286, 'mean_word_length': 4.075196408529742, 'pattern_counts': {'":': 2, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3219961', 'n_tokens_mistral': 5017, 'n_tokens_neox': 4729, 'n_words': 2276}