<commit_before>/*
 * $Id: ArchivalUnitStatus.java,v 1.111 2012-10-15 04:22:20 tlipkis Exp $
 */

/*
 Copyright (c) 2000-2012 Board of Trustees of Leland Stanford Jr. University,
 all rights reserved.
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
 IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 Except as contained in this notice, the name of Stanford University shall not
 be used in advertising or otherwise to promote the sale, use or other dealings
 in this Software without prior written authorization from Stanford University.
 */

package org.lockss.state;

import java.io.*;
import java.util.*;
import java.text.*;
import java.net.MalformedURLException;

import org.lockss.config.Configuration;
import org.lockss.daemon.*;
import org.lockss.daemon.status.*;
import org.lockss.plugin.*;
import org.lockss.util.*;
import org.lockss.app.*;
import org.lockss.crawler.*;
import org.lockss.poller.*;
import org.lockss.poller.v3.*;
import org.lockss.protocol.*;
import org.lockss.repository.*;
import org.lockss.servlet.*;

/**
 * Collect and report the status of the ArchivalUnits
 */
public class ArchivalUnitStatus
  extends BaseLockssDaemonManager implements ConfigurableManager {

  public static final String PREFIX = Configuration.PREFIX + "auStatus.";
  /**
   * The default maximum number of nodes to display in a single page of the ui.
   */
  public static final String PARAM_MAX_NODES_TO_DISPLAY =
    PREFIX + "nodesPerPage";
  static final int DEFAULT_MAX_NODES_TO_DISPLAY = 100;

  /**
   * Node URLs are links to cached content page if true
   */
  public static final String PARAM_CONTENT_IS_LINK =
    PREFIX + "contentUrlIsLink";
  static final boolean DEFAULT_CONTENT_IS_LINK = true;

  /**
   * Include number of AUs needing recrawl in overview if true
   */
  public static final String PARAM_INCLUDE_NEEDS_RECRAWL =
    PREFIX + "includeNeedsRecrawl";
  static final boolean DEFAULT_INCLUDE_NEEDS_RECRAWL = true;

  public static final String SERVICE_STATUS_TABLE_NAME =
      "ArchivalUnitStatusTable";
  public static final String AUIDS_TABLE_NAME = "AuIds";
  public static final String AU_STATUS_TABLE_NAME = "ArchivalUnitTable";
  public static final String NO_AU_PEERS_TABLE_NAME = "NoAuPeers";
  public static final String PEERS_VOTE_TABLE_NAME = "PeerVoteSummary";
  public static final String PEERS_REPAIR_TABLE_NAME = "PeerRepair";
  public static final String FILE_VERSIONS_TABLE_NAME = "FileVersions";
  public static final String AU_DEFINITION_TABLE_NAME = "AuConfiguration";
  public static final String AUS_WITH_URL_TABLE_NAME = "AusWithUrl";


  private static Logger logger = Logger.getLogger("AuStatus");
  private static int defaultNumRows = DEFAULT_MAX_NODES_TO_DISPLAY;
  private static boolean isContentIsLink = DEFAULT_CONTENT_IS_LINK;
  private static boolean includeNeedsRecrawl = DEFAULT_INCLUDE_NEEDS_RECRAWL;
  
  private static final DecimalFormat agreementFormat =
    new DecimalFormat("0.00");

  /**
   * Format 100 times a double to print as a percentage. An input
   * value of 1.0 will produce "100.00".
   * @param a value to convert.
   * @return a String representing the double. 
   */
  static String doubleToPercent(double d) {
    int i = (int)(d * 10000);
    double pc = i / 100.0;
    return agreementFormat.format(pc);
  }

  public void startService() {
    super.startService();

    LockssDaemon theDaemon = getDaemon();
    StatusService statusServ = theDaemon.getStatusService();
    statusServ.registerStatusAccessor(SERVICE_STATUS_TABLE_NAME,
                                      new AuSummary(theDaemon));
    statusServ.registerOverviewAccessor(SERVICE_STATUS_TABLE_NAME,
                                      new Overview(theDaemon));
    statusServ.registerStatusAccessor(AUIDS_TABLE_NAME,
                                      new AuIds(theDaemon));
    statusServ.registerStatusAccessor(AU_STATUS_TABLE_NAME,
                                      new AuStatus(theDaemon));
    statusServ.registerStatusAccessor(NO_AU_PEERS_TABLE_NAME,
                                      new NoAuPeers(theDaemon));
    statusServ.registerStatusAccessor(PEERS_VOTE_TABLE_NAME,
                                      new PeerVoteSummary(theDaemon));
    statusServ.registerStatusAccessor(PEERS_REPAIR_TABLE_NAME,
                                      new PeerRepair(theDaemon));
    statusServ.registerStatusAccessor(FILE_VERSIONS_TABLE_NAME,
                                      new FileVersions(theDaemon));
    statusServ.registerStatusAccessor(AU_DEFINITION_TABLE_NAME,
                                      new AuConfiguration(theDaemon));
    statusServ.registerStatusAccessor(AUS_WITH_URL_TABLE_NAME,
                                      new AusWithUrl(theDaemon));
    logger.debug2("Status accessors registered.");
  }

  public void stopService() {
    // unregister our status accessors
    LockssDaemon theDaemon = getDaemon();
    StatusService statusServ = theDaemon.getStatusService();
    statusServ.unregisterStatusAccessor(SERVICE_STATUS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(NO_AU_PEERS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(AU_STATUS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(PEERS_VOTE_TABLE_NAME);
    statusServ.unregisterStatusAccessor(PEERS_REPAIR_TABLE_NAME);
    statusServ.unregisterStatusAccessor(FILE_VERSIONS_TABLE_NAME);
    logger.debug2("Status accessors unregistered.");
    super.stopService();
  }

  public void setConfig(Configuration config, Configuration oldConfig,
			Configuration.Differences changedKeys) {
    defaultNumRows = config.getInt(PARAM_MAX_NODES_TO_DISPLAY,
                                   DEFAULT_MAX_NODES_TO_DISPLAY);
    isContentIsLink = config.getBoolean(PARAM_CONTENT_IS_LINK,
					DEFAULT_CONTENT_IS_LINK);
    includeNeedsRecrawl = config.getBoolean(PARAM_INCLUDE_NEEDS_RECRAWL,
					    DEFAULT_INCLUDE_NEEDS_RECRAWL);
  }

  static CrawlManagerStatus getCMStatus(LockssDaemon daemon) {
    CrawlManager crawlMgr = daemon.getCrawlManager();
    CrawlManager.StatusSource source = crawlMgr.getStatusSource();
    return source.getStatus();
  }


  static class AuSummary implements StatusAccessor {
    static final String TABLE_TITLE = "Archival Units";

    static final String FOOT_STATUS = "Flags may follow status: C means the AU is complete, D means that the AU is no longer available from the publisher, NS means the AU has no files containing substantial content.";

    static final String FOOT_SIZE = "If blank, size has changed and is being recalculated.  Check again later.";


    private static final List columnDescriptors = ListUtil.list(
      new ColumnDescriptor("AuName", "Volume", ColumnDescriptor.TYPE_STRING),
//       new ColumnDescriptor("AuNodeCount", "Nodes", ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("AuSize", "Content Size",
			   ColumnDescriptor.TYPE_INT, FOOT_SIZE),
      new ColumnDescriptor("DiskUsage", "Disk Usage (MB)",
			   ColumnDescriptor.TYPE_FLOAT, FOOT_SIZE),
      new ColumnDescriptor("Peers", "Peers", ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("AuPolls", "Recent Polls",
                           ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("Damaged", "Status",
                           ColumnDescriptor.TYPE_STRING,
			   FOOT_STATUS),
      new ColumnDescriptor("AuLastPoll", "Last Poll",
                           ColumnDescriptor.TYPE_DATE),
      new ColumnDescriptor("AuLastCrawlAttempt", "Last Crawl Start",
                           ColumnDescriptor.TYPE_DATE),
      new ColumnDescriptor("AuLastCrawlResultMsg", "Last Crawl Result",
                           ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("AuLastCrawl", "Last Successful Crawl",
                           ColumnDescriptor.TYPE_DATE)
      );

    private static final List sortRules =
      ListUtil.list(new
		    StatusTable.SortRule("AuName",
					 CatalogueOrderComparator.SINGLETON));

    private LockssDaemon theDaemon;
    private RepositoryManager repoMgr;
    private CrawlManagerStatus cmStatus;

    AuSummary(LockssDaemon theDaemon) {
      this.theDaemon = theDaemon;
      repoMgr = theDaemon.getRepositoryManager();
      cmStatus = getCMStatus(theDaemon);
    }

    public String getDisplayName() {
      return TABLE_TITLE;
    }

    public void populateTable(StatusTable table)
        throws StatusService.NoSuchTableException {
      List cols = columnDescriptors;
      if (theDaemon.isDetectClockssSubscription()) {
	cols = new ArrayList(cols);
	cols.remove(cols.size() - 1);
	cols.add(new ColumnDescriptor("Subscribed", "Subscribed",
				      ColumnDescriptor.TYPE_STRING));
      }
      table.setColumnDescriptors(cols);
      table.setDefaultSortRules(sortRules);
      Stats stats = new Stats();
      table.setRows(getRows(table, stats));
      table.setSummaryInfo(getSummaryInfo(stats));
    }

    public boolean requiresKey() {
      return false;
    }

    private List getRows(StatusTable table, Stats stats) {
      PluginManager pluginMgr = theDaemon.getPluginManager();

      String key = table.getKey();
      Plugin onlyPlug = null;
      if (key != null && key.startsWith("plugin:")) {
	String[] foo = org.apache.commons.lang.StringUtils.split(key, ":", 2);
	String plugid = foo[1];
	if (!StringUtil.isNullString(plugid)) {
	  onlyPlug = pluginMgr.getPlugin(PluginManager.pluginKeyFromId(plugid));
	  if (onlyPlug != null) {
	    table.setTitle(TABLE_TITLE + " for plugin " +
			   StringUtil.shortName(onlyPlug.getPluginName()));
	  }
	}
      }	
      boolean includeInternalAus =
	table.getOptions().get(StatusTable.OPTION_DEBUG_USER);
      List rowL = new ArrayList();
      Collection<ArchivalUnit> aus;
      if (onlyPlug != null) {
	aus = onlyPlug.getAllAus();
      } else {
	aus = pluginMgr.getAllAus();
      }
      for (ArchivalUnit au : aus) {
	if (!includeInternalAus && pluginMgr.isInternalAu(au)) {
	  continue;
	}
	try {
	  NodeManager nodeMan = theDaemon.getNodeManager(au);
	  rowL.add(makeRow(au, nodeMan));
	  stats.aus++;
	} catch (Exception e) {
	  logger.warning("Unexpected expection building row", e);
	}
      }
      stats.restarting = pluginMgr.getNumAusRestarting();
      return rowL;
    }

    private Map makeRow(ArchivalUnit au, NodeManager nodeMan) {
      AuState auState = nodeMan.getAuState();
      HashMap rowMap = new HashMap();
      PollManager.V3PollStatusAccessor v3status =
        theDaemon.getPollManager().getV3Status();
      // If this is a v3 AU, we cannot access some of the poll 
      // status through the nodestate.  Eventually, this will be totally
      // refactored.
      boolean isV3 = AuUtil.getProtocolVersion(au) == Poll.V3_PROTOCOL;
      //"AuID"
      rowMap.put("AuName", AuStatus.makeAuRef(au.getName(), au.getAuId()));
//       rowMap.put("AuNodeCount", new Integer(-1));
      long contentSize = AuUtil.getAuContentSize(au, false);
      if (contentSize != -1) {
	rowMap.put("AuSize", new Long(contentSize));
      }
      long du = AuUtil.getAuDiskUsage(au, false);
      if (du != -1) {
	rowMap.put("DiskUsage", new Double(((double)du) / (1024*1024)));
      }
      long lastCrawl = auState.getLastCrawlTime();
      long lastAttempt = auState.getLastCrawlAttempt();
      int lastResultCode = auState.getLastCrawlResult();
      String lastResult = auState.getLastCrawlResultMsg();
      
      rowMap.put("AuLastCrawl", new Long(lastCrawl));
      // AuState files that show a successful crawl but no lastAttempt just
      // have uninitialized lastXxx fields.  Display time and status of
      // last successful instead
      if (lastCrawl > 0 && lastAttempt <= 0) {
	lastAttempt = lastCrawl;
	lastResultCode = Crawler.STATUS_SUCCESSFUL;
	lastResult = "Successful";
      }
      rowMap.put("AuLastCrawlAttempt", new Long(lastAttempt));
      Object lastCrawlStatus =
	lastCrawlStatus(au, lastCrawl, lastResultCode, lastResult);
      if (lastCrawlStatus != null) {
	if (lastResultCode == Crawler.STATUS_SUCCESSFUL &&
	    auState.hasNoSubstance()) {
	  lastCrawlStatus =
	    new StatusTable.DisplayedValue(lastCrawlStatus).setFootnote(SingleCrawlStatusAccessor.FOOT_NO_SUBSTANCE_CRAWL_STATUS);
	}
	rowMap.put("AuLastCrawlResultMsg", lastCrawlStatus);
      }

      rowMap.put("Peers", PeerRepair.makeAuRef("peers", au.getAuId()));
      rowMap.put("AuLastPoll", new Long(auState.getLastTopLevelPollTime()));
      
      Object stat;
      if (isV3) {
	int numPolls = v3status.getNumPolls(au.getAuId());
	rowMap.put("AuPolls", pollsRef(new Integer(numPolls), au));
        // Percent damaged.  It's scary to see '0% Agreement' if there's no
        // history, so we just show a friendlier message.
        //
        if (auState.getHighestV3Agreement() < 0 ||
	    auState.getLastTopLevelPollTime() <= 0) {
	  if (cmStatus.isRunningNCCrawl(au)) {
	    stat = new OrderedObject("Crawling", STATUS_ORDER_CRAWLING);
	  } else {
	    if (auState.lastCrawlTime > 0 || AuUtil.isPubDown(au)) {
	      stat = new OrderedObject("Waiting for Poll",
				       STATUS_ORDER_WAIT_POLL);
	    } else {
	      stat = new OrderedObject("Waiting for Crawl",
				       STATUS_ORDER_WAIT_CRAWL);
	    }
          }
        } else {
          stat = agreeStatus(auState.getHighestV3Agreement());
        }
      } else {
        rowMap.put("AuPolls",
                   theDaemon.getStatusService().
                   getReference(PollerStatus.MANAGER_STATUS_TABLE_NAME,
                                au));
	CachedUrlSet auCus = au.getAuCachedUrlSet();
	NodeState topNodeState = nodeMan.getNodeState(auCus);
	stat = topNodeState.hasDamage()
	  ? DAMAGE_STATE_DAMAGED : DAMAGE_STATE_OK;
      }

      boolean isPubDown = AuUtil.isPubDown(au);
      boolean isClosed = AuUtil.isClosed(au);
      boolean noSubstance = auState.hasNoSubstance();
        
      if (isPubDown || isClosed || noSubstance) {
	List flags = new ArrayList();
	if (isClosed) {
	  flags.add("C");
	}
	if (isPubDown) {
	  flags.add("D");
	}
	if (noSubstance) {
	  flags.add("NS");
	}
	String flagStr = StringUtil.separatedString(flags, " (", ",", ")");
	stat = ListUtil.list(stat, flagStr);
      }

      rowMap.put("Damaged", stat);

      if (theDaemon.isDetectClockssSubscription()) {
	rowMap.put("Subscribed",
		   AuUtil.getAuState(au).getClockssSubscriptionStatusString());
      }

      return rowMap;
    }

    String lastCrawlStatus(ArchivalUnit au, long lastCrawl,
			   int lastResultCode, String lastResult) {
      if (AuUtil.isPubDown(au)) {
	if (lastCrawl > 0) {
	  return "No longer crawled";
	} else {
	  return "Never crawled";
	}
      } else {
	if (lastResultCode > 0) {
	  return lastResult;
	}
      }
      return null;
    }

    Object pollsRef(Object val, ArchivalUnit au) {
      return new StatusTable.Reference(val,
				       V3PollStatus.POLLER_STATUS_TABLE_NAME,
				       au.getAuId());
    }

    private List getSummaryInfo(Stats stats) {
      List res = new ArrayList();
      String numaus = StringUtil.numberOfUnits(stats.aus, "Archival Unit",
					       "Archival Units");
      res.add(new StatusTable.SummaryInfo(null,
					  ColumnDescriptor.TYPE_STRING,
					  numaus));
      if (stats.restarting != 0) {
	res.add(new StatusTable.SummaryInfo(null,
					    ColumnDescriptor.TYPE_STRING,
					    stats.restarting + " restarting"));
      }
      int n = repoMgr.sizeCalcQueueLen();
      if (n != 0) {
	res.add(new StatusTable.SummaryInfo(null,
					    ColumnDescriptor.TYPE_STRING,
					    n + " awaiting recalc"));
      }
      return res;
    }
  }

  static class AuIds implements StatusAccessor {
    static final String TABLE_TITLE = "AU Ids";

    private static final List columnDescriptors = ListUtil.list(
      new ColumnDescriptor("AuName", "Volume", ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("AuId", "AU Id", ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("CrawlPool", "Crawl Pool",
			   ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("Publisher", "Publisher",
			   ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("Year", "Year", ColumnDescriptor.TYPE_STRING)
      );

    private static final List<String> defaultCols =
      ListUtil.list("AuName", "AuId");

    private static final List sortRules =
      ListUtil.list(new
		    StatusTable.SortRule("AuName",
					 CatalogueOrderComparator.SINGLETON));

    private LockssDaemon theDaemon;

    AuIds(LockssDaemon theDaemon) {
      this.theDaemon = theDaemon;
    }

    public String getDisplayName() {
      return TABLE_TITLE;
    }

    public void populateTable(StatusTable table)
        throws StatusService.NoSuchTableException {
      table.setColumnDescriptors(columnDescriptors, defaultCols);
      table.setDefaultSortRules(sortRules);
      Stats stats = new Stats();
      table.setRows(getRows(table, stats));
      table.setSummaryInfo(getSummaryInfo(stats));
    }

    public boolean requiresKey() {
      return false;
    }

    private List getRows(StatusTable table,
			 Stats stats) {
      PluginManager pluginMgr = theDaemon.getPluginManager();<commit_msg>The DaemonStatus/Archival Units table now omits the size columns by  default, and comes up quickly.  There's a link on that page to toggle  the size columns (or include "&columns=*" in the request to include  them).<commit_after>/*
 * $Id: ArchivalUnitStatus.java,v 1.112 2013-01-02 20:55:05 tlipkis Exp $
 */

/*
 Copyright (c) 2000-2012 Board of Trustees of Leland Stanford Jr. University,
 all rights reserved.
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
 IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 Except as contained in this notice, the name of Stanford University shall not
 be used in advertising or otherwise to promote the sale, use or other dealings
 in this Software without prior written authorization from Stanford University.
 */

package org.lockss.state;

import java.io.*;
import java.util.*;
import java.text.*;
import java.net.MalformedURLException;

import org.lockss.config.Configuration;
import org.lockss.daemon.*;
import org.lockss.daemon.status.*;
import org.lockss.plugin.*;
import org.lockss.util.*;
import org.lockss.app.*;
import org.lockss.crawler.*;
import org.lockss.poller.*;
import org.lockss.poller.v3.*;
import org.lockss.protocol.*;
import org.lockss.repository.*;
import org.lockss.servlet.*;

/**
 * Collect and report the status of the ArchivalUnits
 */
public class ArchivalUnitStatus
  extends BaseLockssDaemonManager implements ConfigurableManager {

  public static final String PREFIX = Configuration.PREFIX + "auStatus.";
  /**
   * The default maximum number of nodes to display in a single page of the ui.
   */
  public static final String PARAM_MAX_NODES_TO_DISPLAY =
    PREFIX + "nodesPerPage";
  static final int DEFAULT_MAX_NODES_TO_DISPLAY = 100;

  /**
   * Node URLs are links to cached content page if true
   */
  public static final String PARAM_CONTENT_IS_LINK =
    PREFIX + "contentUrlIsLink";
  static final boolean DEFAULT_CONTENT_IS_LINK = true;

  /**
   * Include number of AUs needing recrawl in overview if true
   */
  public static final String PARAM_INCLUDE_NEEDS_RECRAWL =
    PREFIX + "includeNeedsRecrawl";
  static final boolean DEFAULT_INCLUDE_NEEDS_RECRAWL = true;

  public static final String SERVICE_STATUS_TABLE_NAME =
      "ArchivalUnitStatusTable";
  public static final String AUIDS_TABLE_NAME = "AuIds";
  public static final String AU_STATUS_TABLE_NAME = "ArchivalUnitTable";
  public static final String NO_AU_PEERS_TABLE_NAME = "NoAuPeers";
  public static final String PEERS_VOTE_TABLE_NAME = "PeerVoteSummary";
  public static final String PEERS_REPAIR_TABLE_NAME = "PeerRepair";
  public static final String FILE_VERSIONS_TABLE_NAME = "FileVersions";
  public static final String AU_DEFINITION_TABLE_NAME = "AuConfiguration";
  public static final String AUS_WITH_URL_TABLE_NAME = "AusWithUrl";


  private static Logger logger = Logger.getLogger("AuStatus");
  private static int defaultNumRows = DEFAULT_MAX_NODES_TO_DISPLAY;
  private static boolean isContentIsLink = DEFAULT_CONTENT_IS_LINK;
  private static boolean includeNeedsRecrawl = DEFAULT_INCLUDE_NEEDS_RECRAWL;
  
  private static final DecimalFormat agreementFormat =
    new DecimalFormat("0.00");

  /**
   * Format 100 times a double to print as a percentage. An input
   * value of 1.0 will produce "100.00".
   * @param a value to convert.
   * @return a String representing the double. 
   */
  static String doubleToPercent(double d) {
    int i = (int)(d * 10000);
    double pc = i / 100.0;
    return agreementFormat.format(pc);
  }

  public void startService() {
    super.startService();

    LockssDaemon theDaemon = getDaemon();
    StatusService statusServ = theDaemon.getStatusService();
    statusServ.registerStatusAccessor(SERVICE_STATUS_TABLE_NAME,
                                      new AuSummary(theDaemon));
    statusServ.registerOverviewAccessor(SERVICE_STATUS_TABLE_NAME,
                                      new Overview(theDaemon));
    statusServ.registerStatusAccessor(AUIDS_TABLE_NAME,
                                      new AuIds(theDaemon));
    statusServ.registerStatusAccessor(AU_STATUS_TABLE_NAME,
                                      new AuStatus(theDaemon));
    statusServ.registerStatusAccessor(NO_AU_PEERS_TABLE_NAME,
                                      new NoAuPeers(theDaemon));
    statusServ.registerStatusAccessor(PEERS_VOTE_TABLE_NAME,
                                      new PeerVoteSummary(theDaemon));
    statusServ.registerStatusAccessor(PEERS_REPAIR_TABLE_NAME,
                                      new PeerRepair(theDaemon));
    statusServ.registerStatusAccessor(FILE_VERSIONS_TABLE_NAME,
                                      new FileVersions(theDaemon));
    statusServ.registerStatusAccessor(AU_DEFINITION_TABLE_NAME,
                                      new AuConfiguration(theDaemon));
    statusServ.registerStatusAccessor(AUS_WITH_URL_TABLE_NAME,
                                      new AusWithUrl(theDaemon));
    logger.debug2("Status accessors registered.");
  }

  public void stopService() {
    // unregister our status accessors
    LockssDaemon theDaemon = getDaemon();
    StatusService statusServ = theDaemon.getStatusService();
    statusServ.unregisterStatusAccessor(SERVICE_STATUS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(NO_AU_PEERS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(AU_STATUS_TABLE_NAME);
    statusServ.unregisterStatusAccessor(PEERS_VOTE_TABLE_NAME);
    statusServ.unregisterStatusAccessor(PEERS_REPAIR_TABLE_NAME);
    statusServ.unregisterStatusAccessor(FILE_VERSIONS_TABLE_NAME);
    logger.debug2("Status accessors unregistered.");
    super.stopService();
  }

  public void setConfig(Configuration config, Configuration oldConfig,
			Configuration.Differences changedKeys) {
    defaultNumRows = config.getInt(PARAM_MAX_NODES_TO_DISPLAY,
                                   DEFAULT_MAX_NODES_TO_DISPLAY);
    isContentIsLink = config.getBoolean(PARAM_CONTENT_IS_LINK,
					DEFAULT_CONTENT_IS_LINK);
    includeNeedsRecrawl = config.getBoolean(PARAM_INCLUDE_NEEDS_RECRAWL,
					    DEFAULT_INCLUDE_NEEDS_RECRAWL);
  }

  static CrawlManagerStatus getCMStatus(LockssDaemon daemon) {
    CrawlManager crawlMgr = daemon.getCrawlManager();
    CrawlManager.StatusSource source = crawlMgr.getStatusSource();
    return source.getStatus();
  }


  /** By default the AuSummary table omits the size columns.  Specify
   * columns=* to include them */
  static final String DEFAULT_AU_SUMMARY_COLUMNS = "-AuSize;DiskUsage";

  static class AuSummary implements StatusAccessor {
    static final String TABLE_TITLE = "Archival Units";

    static final String FOOT_STATUS = "Flags may follow status: C means the AU is complete, D means that the AU is no longer available from the publisher, NS means the AU has no files containing substantial content.";

    static final String FOOT_SIZE = "If blank, size has changed and is being recalculated.  Check again later.";


    private static final List columnDescriptors = ListUtil.list(
      new ColumnDescriptor("AuName", "Volume", ColumnDescriptor.TYPE_STRING),
//       new ColumnDescriptor("AuNodeCount", "Nodes", ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("AuSize", "Content Size",
			   ColumnDescriptor.TYPE_INT, FOOT_SIZE),
      new ColumnDescriptor("DiskUsage", "Disk Usage (MB)",
			   ColumnDescriptor.TYPE_FLOAT, FOOT_SIZE),
      new ColumnDescriptor("Peers", "Peers", ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("AuPolls", "Recent Polls",
                           ColumnDescriptor.TYPE_INT),
      new ColumnDescriptor("Damaged", "Status",
                           ColumnDescriptor.TYPE_STRING,
			   FOOT_STATUS),
      new ColumnDescriptor("AuLastPoll", "Last Poll",
                           ColumnDescriptor.TYPE_DATE),
      new ColumnDescriptor("AuLastCrawlAttempt", "Last Crawl Start",
                           ColumnDescriptor.TYPE_DATE),
      new ColumnDescriptor("AuLastCrawlResultMsg", "Last Crawl Result",
                           ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("AuLastCrawl", "Last Successful Crawl",
                           ColumnDescriptor.TYPE_DATE)
      );

    private static final List sortRules =
      ListUtil.list(new
		    StatusTable.SortRule("AuName",
					 CatalogueOrderComparator.SINGLETON));

    private LockssDaemon theDaemon;
    private RepositoryManager repoMgr;
    private CrawlManagerStatus cmStatus;

    AuSummary(LockssDaemon theDaemon) {
      this.theDaemon = theDaemon;
      repoMgr = theDaemon.getRepositoryManager();
      cmStatus = getCMStatus(theDaemon);
    }

    public String getDisplayName() {
      return TABLE_TITLE;
    }

    public boolean requiresKey() {
      return false;
    }

    public void populateTable(StatusTable table)
        throws StatusService.NoSuchTableException {
      List cols = columnDescriptors;
      if (theDaemon.isDetectClockssSubscription()) {
	cols = new ArrayList(cols);
	cols.remove(cols.size() - 1);
	cols.add(new ColumnDescriptor("Subscribed", "Subscribed",
				      ColumnDescriptor.TYPE_STRING));
      }
      table.setColumnDescriptors(cols, DEFAULT_AU_SUMMARY_COLUMNS);
      table.setDefaultSortRules(sortRules);
      Set<String> inclCols = new HashSet<String>();
      for (ColumnDescriptor cd : table.getColumnDescriptors()) {
	inclCols.add(cd.getColumnName());
      }
      Stats stats = new Stats();
      table.setRows(getRows(table, inclCols, stats));
      table.setSummaryInfo(getSummaryInfo(table, inclCols, stats));
    }

    private List getRows(StatusTable table, Set<String> inclCols, Stats stats) {
      PluginManager pluginMgr = theDaemon.getPluginManager();

      String key = table.getKey();
      Plugin onlyPlug = null;
      if (key != null && key.startsWith("plugin:")) {
	String[] foo = org.apache.commons.lang.StringUtils.split(key, ":", 2);
	String plugid = foo[1];
	if (!StringUtil.isNullString(plugid)) {
	  onlyPlug = pluginMgr.getPlugin(PluginManager.pluginKeyFromId(plugid));
	  if (onlyPlug != null) {
	    table.setTitle(TABLE_TITLE + " for plugin " +
			   StringUtil.shortName(onlyPlug.getPluginName()));
	  }
	}
      }	
      boolean includeInternalAus =
	table.getOptions().get(StatusTable.OPTION_DEBUG_USER);
      List rowL = new ArrayList();
      Collection<ArchivalUnit> aus;
      if (onlyPlug != null) {
	aus = onlyPlug.getAllAus();
      } else {
	aus = pluginMgr.getAllAus();
      }
      for (ArchivalUnit au : aus) {
	if (!includeInternalAus && pluginMgr.isInternalAu(au)) {
	  continue;
	}
	try {
	  NodeManager nodeMan = theDaemon.getNodeManager(au);
	  rowL.add(makeRow(au, nodeMan, inclCols));
	  stats.aus++;
	} catch (Exception e) {
	  logger.warning("Unexpected expection building row", e);
	}
      }
      stats.restarting = pluginMgr.getNumAusRestarting();
      return rowL;
    }

    private Map makeRow(ArchivalUnit au, NodeManager nodeMan, Set inclCols) {
      AuState auState = nodeMan.getAuState();
      HashMap rowMap = new HashMap();
      PollManager.V3PollStatusAccessor v3status =
        theDaemon.getPollManager().getV3Status();
      // If this is a v3 AU, we cannot access some of the poll 
      // status through the nodestate.  Eventually, this will be totally
      // refactored.
      boolean isV3 = AuUtil.getProtocolVersion(au) == Poll.V3_PROTOCOL;
      //"AuID"
      rowMap.put("AuName", AuStatus.makeAuRef(au.getName(), au.getAuId()));
//       rowMap.put("AuNodeCount", new Integer(-1));
      if (inclCols.contains("AuSize")) {
	long contentSize = AuUtil.getAuContentSize(au, false);
	if (contentSize != -1) {
	  rowMap.put("AuSize", new Long(contentSize));
	}
      }
      if (inclCols.contains("DiskUsage")) {
	long du = AuUtil.getAuDiskUsage(au, false);
	if (du != -1) {
	  rowMap.put("DiskUsage", new Double(((double)du) / (1024*1024)));
	}
      }
      long lastCrawl = auState.getLastCrawlTime();
      long lastAttempt = auState.getLastCrawlAttempt();
      int lastResultCode = auState.getLastCrawlResult();
      String lastResult = auState.getLastCrawlResultMsg();
      
      rowMap.put("AuLastCrawl", new Long(lastCrawl));
      // AuState files that show a successful crawl but no lastAttempt just
      // have uninitialized lastXxx fields.  Display time and status of
      // last successful instead
      if (lastCrawl > 0 && lastAttempt <= 0) {
	lastAttempt = lastCrawl;
	lastResultCode = Crawler.STATUS_SUCCESSFUL;
	lastResult = "Successful";
      }
      rowMap.put("AuLastCrawlAttempt", new Long(lastAttempt));
      Object lastCrawlStatus =
	lastCrawlStatus(au, lastCrawl, lastResultCode, lastResult);
      if (lastCrawlStatus != null) {
	if (lastResultCode == Crawler.STATUS_SUCCESSFUL &&
	    auState.hasNoSubstance()) {
	  lastCrawlStatus =
	    new StatusTable.DisplayedValue(lastCrawlStatus).setFootnote(SingleCrawlStatusAccessor.FOOT_NO_SUBSTANCE_CRAWL_STATUS);
	}
	rowMap.put("AuLastCrawlResultMsg", lastCrawlStatus);
      }

      rowMap.put("Peers", PeerRepair.makeAuRef("peers", au.getAuId()));
      rowMap.put("AuLastPoll", new Long(auState.getLastTopLevelPollTime()));
      
      Object stat;
      if (isV3) {
	int numPolls = v3status.getNumPolls(au.getAuId());
	rowMap.put("AuPolls", pollsRef(new Integer(numPolls), au));
        // Percent damaged.  It's scary to see '0% Agreement' if there's no
        // history, so we just show a friendlier message.
        //
        if (auState.getHighestV3Agreement() < 0 ||
	    auState.getLastTopLevelPollTime() <= 0) {
	  if (cmStatus.isRunningNCCrawl(au)) {
	    stat = new OrderedObject("Crawling", STATUS_ORDER_CRAWLING);
	  } else {
	    if (auState.lastCrawlTime > 0 || AuUtil.isPubDown(au)) {
	      stat = new OrderedObject("Waiting for Poll",
				       STATUS_ORDER_WAIT_POLL);
	    } else {
	      stat = new OrderedObject("Waiting for Crawl",
				       STATUS_ORDER_WAIT_CRAWL);
	    }
          }
        } else {
          stat = agreeStatus(auState.getHighestV3Agreement());
        }
      } else {
        rowMap.put("AuPolls",
                   theDaemon.getStatusService().
                   getReference(PollerStatus.MANAGER_STATUS_TABLE_NAME,
                                au));
	CachedUrlSet auCus = au.getAuCachedUrlSet();
	NodeState topNodeState = nodeMan.getNodeState(auCus);
	stat = topNodeState.hasDamage()
	  ? DAMAGE_STATE_DAMAGED : DAMAGE_STATE_OK;
      }

      boolean isPubDown = AuUtil.isPubDown(au);
      boolean isClosed = AuUtil.isClosed(au);
      boolean noSubstance = auState.hasNoSubstance();
        
      if (isPubDown || isClosed || noSubstance) {
	List flags = new ArrayList();
	if (isClosed) {
	  flags.add("C");
	}
	if (isPubDown) {
	  flags.add("D");
	}
	if (noSubstance) {
	  flags.add("NS");
	}
	String flagStr = StringUtil.separatedString(flags, " (", ",", ")");
	stat = ListUtil.list(stat, flagStr);
      }

      rowMap.put("Damaged", stat);

      if (theDaemon.isDetectClockssSubscription()) {
	rowMap.put("Subscribed",
		   AuUtil.getAuState(au).getClockssSubscriptionStatusString());
      }

      return rowMap;
    }

    String lastCrawlStatus(ArchivalUnit au, long lastCrawl,
			   int lastResultCode, String lastResult) {
      if (AuUtil.isPubDown(au)) {
	if (lastCrawl > 0) {
	  return "No longer crawled";
	} else {
	  return "Never crawled";
	}
      } else {
	if (lastResultCode > 0) {
	  return lastResult;
	}
      }
      return null;
    }

    Object pollsRef(Object val, ArchivalUnit au) {
      return new StatusTable.Reference(val,
				       V3PollStatus.POLLER_STATUS_TABLE_NAME,
				       au.getAuId());
    }

    private List getSummaryInfo(StatusTable table, Set<String> inclCols,
				Stats stats) {
      List res = new ArrayList();
      String numaus = StringUtil.numberOfUnits(stats.aus, "Archival Unit",
					       "Archival Units");
      res.add(new StatusTable.SummaryInfo(null,
					  ColumnDescriptor.TYPE_STRING,
					  numaus));
      if (stats.restarting != 0) {
	res.add(new StatusTable.SummaryInfo(null,
					    ColumnDescriptor.TYPE_STRING,
					    stats.restarting + " restarting"));
      }
      if (inclCols.contains("AuSize") || inclCols.contains("DiskUsage")) {
	int n = repoMgr.sizeCalcQueueLen();
	if (n != 0) {
	  res.add(new StatusTable.SummaryInfo(null,
					      ColumnDescriptor.TYPE_STRING,
					      n + " awaiting recalc"));
	}
	StatusTable.Reference hideSize = 
	  new StatusTable.Reference("Hide AU Sizes",
				    ArchivalUnitStatus.SERVICE_STATUS_TABLE_NAME,
				    table.getKey());
	res.add(new StatusTable.SummaryInfo(null,
					    ColumnDescriptor.TYPE_STRING,
					    hideSize));
      } else {
	StatusTable.Reference showSize = 
	  new StatusTable.Reference("Show AU Sizes",
				    ArchivalUnitStatus.SERVICE_STATUS_TABLE_NAME,
				    table.getKey());
	showSize.setProperty("columns", "*");
	res.add(new StatusTable.SummaryInfo(null,
					    ColumnDescriptor.TYPE_STRING,
					    showSize));
      }
      return res;
    }
  }

  static class AuIds implements StatusAccessor {
    static final String TABLE_TITLE = "AU Ids";

    private static final List columnDescriptors = ListUtil.list(
      new ColumnDescriptor("AuName", "Volume", ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("AuId", "AU Id", ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("CrawlPool", "Crawl Pool",
			   ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("Publisher", "Publisher",
			   ColumnDescriptor.TYPE_STRING),
      new ColumnDescriptor("Year", "Year", ColumnDescriptor.TYPE_STRING)
      );

    private static final List<String> defaultCols =
      ListUtil.list("AuName", "AuId");

    private static final List sortRules =
      ListUtil.list(new
		    StatusTable.SortRule("AuName",
					 CatalogueOrderComparator.SINGLETON));

    private LockssDaemon theDaemon;

    AuIds(LockssDaemon theDaemon) {
      this.theDaemon = theDaemon;
    }

    public String getDisplayName() {
      return TABLE_TITLE;
    }

    public boolean requiresKey() {
      return false;
    }

    public void populateTable(StatusTable table)
        throws StatusService.NoSuchTableException {
      table.setColumnDescriptors(columnDescriptors, defaultCols);
      table.setDefaultSortRules(sortRules);
      Stats stats = new Stats();
      table.setRows(getRows(table, stats));
      table.setSummaryInfo(getSummaryInfo(stats));
    }

    private List getRows(StatusTable table,
			 Stats stats) {
      PluginManager pluginMgr = theDaemon.getPluginManager();
{'fraction_non_alphanumeric': 0.09347814484709399, 'fraction_numerical': 0.004749959972247425, 'mean_word_length': 3.292668957617411, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5991417', 'n_tokens_mistral': 11604, 'n_tokens_neox': 10786, 'n_words': 2663}