<commit_before>/*
 * Copyright (c) 2007 Regents of the University of California
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the University of California, Berkeley nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package edu.berkeley.compbio.ml.strings;

import com.davidsoergel.dsutils.AbstractGenericFactoryAware;
import com.davidsoergel.dsutils.ArrayUtils;
import com.davidsoergel.stats.DistributionException;
import com.davidsoergel.stats.DistributionProcessorException;
import com.davidsoergel.stats.Multinomial;
import edu.berkeley.compbio.ml.cluster.AdditiveClusterable;
import edu.berkeley.compbio.ml.cluster.Clusterable;
import org.apache.commons.lang.NotImplementedException;
import org.apache.log4j.Logger;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class RonPSTTest
	{
	private static Logger logger = Logger.getLogger(RonPSTTest.class);

	@BeforeMethod
	public void setUp() throws Exception
		{
		/*	ThreadLocalRun.removeInstance();
		new ThreadLocalRun()
		{
		public String getVersion()
			{
			return null;//To change body of implemented methods use File | Settings | File Templates.
			}

		public void run() throws RunUnsuccessfulException
			{
			//To change body of implemented methods use File | Settings | File Templates.
			}
		};*/
		//	Map<String, Object> props = new HashMap<String, Object>();

		////props.put("edu.berkeley.compbio.ml.strings.KneserNeyPSTSmoother.smoothFactor", "0.1");

		//	HierarchicalTypedPropertyNode n = PropertyConsumerClassParser.parseRootContextClass(StubSequenceFragmentIterator.class);
		//	MapToHierarchicalTypedPropertyNodeAdapter.mergeInto(n, props);
		//	stubSequenceFragmentIteratorFactory = new PropertyConsumerFactory<StubSequenceFragmentIterator>(n);

		}

	@Test
	public void afterCompleteEveryNodeHasTransitionsForEverySymbolOrNone()
			throws SequenceSpectrumException, DistributionException
		{
		RonPST pst = createSimplePST();

		byte[] alphabet = pst.getAlphabet();
		assert allNodesAreCompleteOrEmpty(pst, alphabet.length);
		}

	private RonPST createSimplePST() throws SequenceSpectrumException
		{
		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 1.1, 0.01, 4, ss);
		//pst.completeAndCopyProbsFrom(ss);
		return pst;
		}

	private boolean allNodesAreCompleteOrEmpty(MarkovTreeNode node, int maxWidth)
		{
		if (!nodeIsCompleteOrEmpty(node, maxWidth))
			{
			return false;
			}
		for (MarkovTreeNode child : node.getChildren().values())
			{
			if (!allNodesAreCompleteOrEmpty(child, maxWidth))
				{
				return false;
				}
			}
		return true;
		}

	private boolean nodeIsCompleteOrEmpty(MarkovTreeNode node, int maxWidth)
		{
		int width = node.getChildren().size();
		if (width != 0 && width != maxWidth)
			{
			return false;
			}
		return true;
		}


	@Test
	public void maxDepthIsCalculated() throws SequenceSpectrumException
		{
		RonPST pst = createSimplePST();
		assert pst.getMaxDepth() == 2;
		}


	@Test
	public void conditionalProbabilitiesAreGivenBasedOnLongestAvailableSuffix() throws SequenceSpectrumException
		{
		RonPST pst = createSimplePST();

		assert pst.conditionalProbability((byte) 'a', new byte[]{'d', 'a', 'b', 'a', 'b'}) == pst
				.conditionalProbability((byte) 'a', new byte[]{'a', 'b'});
		assert pst.conditionalProbability((byte) 'a', new byte[]{'a', 'b'}) != pst
				.conditionalProbability((byte) 'a', new byte[]{'b'});
		assert pst.conditionalProbability((byte) 'a', new byte[]{'a', 'b'}) != pst
				.conditionalProbability((byte) 'a', new byte[]{'a'});
		}

	@Test
	public void highRatioThresholdProducesShallowTree() throws SequenceSpectrumException
		{
		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 500, 0.01, 4, ss);
		//	pst.completeAndCopyProbsFrom(ss);

		assert pst.getMaxDepth() == 1;
		}


	@Test
	public void lowRatioThresholdProducesDeepTree() throws SequenceSpectrumException
		{
		// todo improve by making a deeper tree to test

		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 1, 0.01, 4, ss);
		//		pst.completeAndCopyProbsFrom(ss);

		assert pst.getMaxDepth() == 2;
		}

	public static SequenceSpectrum createStubSimpleSequenceSpectrum() throws SequenceSpectrumException
		{
		return new StubSequenceSpectrum();<commit_msg>Test failure was just wrong<commit_after>/*
 * Copyright (c) 2007 Regents of the University of California
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the University of California, Berkeley nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package edu.berkeley.compbio.ml.strings;

import com.davidsoergel.dsutils.AbstractGenericFactoryAware;
import com.davidsoergel.dsutils.ArrayUtils;
import com.davidsoergel.stats.DistributionException;
import com.davidsoergel.stats.DistributionProcessorException;
import com.davidsoergel.stats.Multinomial;
import edu.berkeley.compbio.ml.cluster.AdditiveClusterable;
import edu.berkeley.compbio.ml.cluster.Clusterable;
import org.apache.commons.lang.NotImplementedException;
import org.apache.log4j.Logger;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class RonPSTTest
	{
	private static Logger logger = Logger.getLogger(RonPSTTest.class);

	@BeforeMethod
	public void setUp() throws Exception
		{
		/*	ThreadLocalRun.removeInstance();
		new ThreadLocalRun()
		{
		public String getVersion()
			{
			return null;//To change body of implemented methods use File | Settings | File Templates.
			}

		public void run() throws RunUnsuccessfulException
			{
			//To change body of implemented methods use File | Settings | File Templates.
			}
		};*/
		//	Map<String, Object> props = new HashMap<String, Object>();

		////props.put("edu.berkeley.compbio.ml.strings.KneserNeyPSTSmoother.smoothFactor", "0.1");

		//	HierarchicalTypedPropertyNode n = PropertyConsumerClassParser.parseRootContextClass(StubSequenceFragmentIterator.class);
		//	MapToHierarchicalTypedPropertyNodeAdapter.mergeInto(n, props);
		//	stubSequenceFragmentIteratorFactory = new PropertyConsumerFactory<StubSequenceFragmentIterator>(n);

		}

	@Test
	public void afterCompleteEveryNodeHasTransitionsForEverySymbolOrNone()
			throws SequenceSpectrumException, DistributionException
		{
		RonPST pst = createSimplePST();

		byte[] alphabet = pst.getAlphabet();
		assert allNodesAreCompleteOrEmpty(pst, alphabet.length);
		}

	private RonPST createSimplePST() throws SequenceSpectrumException
		{
		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 1.1, 0.01, 4, ss);
		//pst.completeAndCopyProbsFrom(ss);
		return pst;
		}

	private boolean allNodesAreCompleteOrEmpty(MarkovTreeNode node, int maxWidth)
		{
		if (!nodeIsCompleteOrEmpty(node, maxWidth))
			{
			return false;
			}
		for (MarkovTreeNode child : node.getChildren().values())
			{
			if (!allNodesAreCompleteOrEmpty(child, maxWidth))
				{
				return false;
				}
			}
		return true;
		}

	private boolean nodeIsCompleteOrEmpty(MarkovTreeNode node, int maxWidth)
		{
		int width = node.getChildren().size();
		if (width != 0 && width != maxWidth)
			{
			return false;
			}
		return true;
		}


	@Test
	public void maxDepthIsCalculated() throws SequenceSpectrumException
		{
		RonPST pst = createSimplePST();
		assert pst.getMaxDepth() == 2;
		}


	@Test
	public void conditionalProbabilitiesAreGivenBasedOnLongestAvailableSuffix() throws SequenceSpectrumException
		{
		RonPST pst = createSimplePST();

		// really should use a 3-level pst for this

		assert pst.conditionalProbability((byte) 'a', new byte[]{'d', 'a', 'b', 'a', 'b'}) == pst
				.conditionalProbability((byte) 'a', new byte[]{'a', 'b'});
		assert pst.conditionalProbability((byte) 'a', new byte[]{'a', 'b'}) == pst
				.conditionalProbability((byte) 'a', new byte[]{'b'});
		assert pst.conditionalProbability((byte) 'a', new byte[]{'b'}) != pst
				.conditionalProbability((byte) 'a', new byte[]{});
		assert pst.conditionalProbability((byte) 'a', new byte[]{'a', 'b'}) != pst
				.conditionalProbability((byte) 'a', new byte[]{'a'});
		}

	@Test
	public void highRatioThresholdProducesShallowTree() throws SequenceSpectrumException
		{
		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 500, 0.01, 4, ss);
		//	pst.completeAndCopyProbsFrom(ss);

		assert pst.getMaxDepth() == 1;
		}


	@Test
	public void lowRatioThresholdProducesDeepTree() throws SequenceSpectrumException
		{
		// todo improve by making a deeper tree to test

		SequenceSpectrum ss = createStubSimpleSequenceSpectrum();
		RonPST pst = new RonPST(0.0001, 0, 1, 0.01, 4, ss);
		//		pst.completeAndCopyProbsFrom(ss);

		assert pst.getMaxDepth() == 2;
		}

	public static SequenceSpectrum createStubSimpleSequenceSpectrum() throws SequenceSpectrumException
		{
		return new StubSequenceSpectrum();
{'fraction_non_alphanumeric': 0.09615707333445209, 'fraction_numerical': 0.007971136096660513, 'mean_word_length': 4.582669789227166, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1371112', 'n_tokens_mistral': 3967, 'n_tokens_neox': 3448, 'n_words': 1057}