<commit_before>package thredds.tds.ethan;

import junit.framework.*;

import java.util.*;
import java.io.IOException;
import java.io.PrintStream;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.net.URISyntaxException;

import thredds.catalog.*;
import thredds.catalog.crawl.CatalogCrawler;
import thredds.catalog.query.DqcFactory;
import thredds.catalog.query.QueryCapability;
import thredds.datatype.DateType;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.TypedDataset;
import ucar.nc2.dt.TypedDatasetFactory;
import ucar.nc2.thredds.ThreddsDataFactory;
import ucar.unidata.util.DateUtil;

/**
 * _more_
 *
 * @author edavis
 * @since Feb 15, 2007 10:10:08 PM
 */
public class TestAll extends TestCase
{
  public static Test suite()
  {
    TestSuite suite = new TestSuite();

    String tdsTestLevel = System.getProperty( "thredds.tds.test.level", "ping-catalogs" );
    System.out.println( "Test level: " + tdsTestLevel );

    if ( tdsTestLevel.equalsIgnoreCase( "ping-mlode" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      suite.addTestSuite( TestTdsPingMotherlode.class );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "ping-catalogs" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testPingCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs-and1DsPerCollection" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOpenOneDatasetInEachCollection" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "ping-idd" ) )
    {
      System.setProperty( "thredds.tds.test.catalogs", "catalog.xml,idd/models.xml" );
      suite.addTest( new TestAll( "testPingCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs-oneLevelDeep" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOneLevelDeep" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-topcatalog" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      System.setProperty( "thredds.tds.test.catalogs", "topcatalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOneLevelDeep" ) );
    }
    else
    {
      suite.addTestSuite( thredds.tds.ethan.TestTdsPingMotherlode.class );
    }


    return suite;
  }

  private boolean showDebug = false;

  private String host = "motherlode.ucar.edu:8080";
  private String[] catalogList;

  private String targetTdsUrl;

  public TestAll( String name )
  {
    super( name );
  }

  protected void setUp()
  {
    host = System.getProperty( "thredds.tds.test.server", host );
    targetTdsUrl = "http://" + host + "/thredds/";

    showDebug = Boolean.parseBoolean( System.getProperty( "thredds.tds.test.showDebug", "false" ) );

    String catalogListString = System.getProperty( "thredds.tds.test.catalogs", null );
    if ( catalogListString == null )
      catalogListString = System.getProperty( "thredds.tds.test.catalog", "catalog.xml" );
    catalogList = catalogListString.split( "," );
  }

  public void testPingCatalogs()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= null != TestAll.openAndValidateCatalog( targetTdsUrl + catalogList[i], msg, showDebug );
    }
    assertTrue( "Ping failed on catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogs()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.openAndValidateCatalogTree( targetTdsUrl + catalogList[i], msg, true );
    }
    assertTrue( "Invalid catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogsOneLevelDeep()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.openAndValidateCatalogOneLevelDeep( targetTdsUrl + catalogList[i], msg, false );
    }

    assertTrue( "Invalid catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogsOpenOneDatasetInEachCollection()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.crawlCatalogOpenRandomDataset( targetTdsUrl + catalogList[i], msg );
    }

    assertTrue( "Failed to open dataset(s):\n" + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }<commit_msg>Clarify failure message in testCrawlCatalogsOpenOneDatasetInEachCollection().<commit_after>package thredds.tds.ethan;

import junit.framework.*;

import java.util.*;
import java.io.IOException;
import java.io.PrintStream;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.net.URISyntaxException;

import thredds.catalog.*;
import thredds.catalog.crawl.CatalogCrawler;
import thredds.catalog.query.DqcFactory;
import thredds.catalog.query.QueryCapability;
import thredds.datatype.DateType;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.TypedDataset;
import ucar.nc2.dt.TypedDatasetFactory;
import ucar.nc2.thredds.ThreddsDataFactory;
import ucar.unidata.util.DateUtil;

/**
 * _more_
 *
 * @author edavis
 * @since Feb 15, 2007 10:10:08 PM
 */
public class TestAll extends TestCase
{
  public static Test suite()
  {
    TestSuite suite = new TestSuite();

    String tdsTestLevel = System.getProperty( "thredds.tds.test.level", "ping-catalogs" );
    System.out.println( "Test level: " + tdsTestLevel );

    if ( tdsTestLevel.equalsIgnoreCase( "ping-mlode" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      suite.addTestSuite( TestTdsPingMotherlode.class );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "ping-catalogs" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testPingCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs-and1DsPerCollection" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOpenOneDatasetInEachCollection" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "ping-idd" ) )
    {
      System.setProperty( "thredds.tds.test.catalogs", "catalog.xml,idd/models.xml" );
      suite.addTest( new TestAll( "testPingCatalogs" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-catalogs-oneLevelDeep" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      //System.setProperty( "thredds.tds.test.catalogs", "catalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOneLevelDeep" ) );
    }
    else if ( tdsTestLevel.equalsIgnoreCase( "crawl-topcatalog" ) )
    {
      //System.setProperty( "thredds.tds.test.server", "motherlode.ucar.edu:8080" );
      System.setProperty( "thredds.tds.test.catalogs", "topcatalog.xml" );
      suite.addTest( new TestAll( "testCrawlCatalogsOneLevelDeep" ) );
    }
    else
    {
      suite.addTestSuite( thredds.tds.ethan.TestTdsPingMotherlode.class );
    }


    return suite;
  }

  private boolean showDebug = false;

  private String host = "motherlode.ucar.edu:8080";
  private String[] catalogList;

  private String targetTdsUrl;

  public TestAll( String name )
  {
    super( name );
  }

  protected void setUp()
  {
    host = System.getProperty( "thredds.tds.test.server", host );
    targetTdsUrl = "http://" + host + "/thredds/";

    showDebug = Boolean.parseBoolean( System.getProperty( "thredds.tds.test.showDebug", "false" ) );

    String catalogListString = System.getProperty( "thredds.tds.test.catalogs", null );
    if ( catalogListString == null )
      catalogListString = System.getProperty( "thredds.tds.test.catalog", "catalog.xml" );
    catalogList = catalogListString.split( "," );
  }

  public void testPingCatalogs()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= null != TestAll.openAndValidateCatalog( targetTdsUrl + catalogList[i], msg, showDebug );
    }
    assertTrue( "Ping failed on catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogs()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.openAndValidateCatalogTree( targetTdsUrl + catalogList[i], msg, true );
    }
    assertTrue( "Invalid catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogsOneLevelDeep()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.openAndValidateCatalogOneLevelDeep( targetTdsUrl + catalogList[i], msg, false );
    }

    assertTrue( "Invalid catalog(s): " + msg.toString(),
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
  }

  public void testCrawlCatalogsOpenOneDatasetInEachCollection()
  {
    boolean pass = true;
    StringBuffer msg = new StringBuffer();

    for ( int i = 0; i < catalogList.length; i++ )
    {
      pass &= TestAll.crawlCatalogOpenRandomDataset( targetTdsUrl + catalogList[i], msg );
    }

    assertTrue( "Failed to open dataset(s):\n========================================\n" +
                msg.toString() + "\n========================================\n",
                pass );

    if ( msg.length() > 0 )
    {
      System.out.println( msg.toString() );
    }
{'fraction_non_alphanumeric': 0.14540526361808168, 'fraction_numerical': 0.009268164728512721, 'mean_word_length': 3.1487123685165033, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2973175', 'n_tokens_mistral': 3980, 'n_tokens_neox': 3855, 'n_words': 699}