<commit_before>/*
 * Copyright <NAME> 2014, All rights reserved.
 *
 * This code is licensed under a standard 3-clause BSD license:
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice, 
 *    this list of conditions and the following disclaimer.
 * 
 *  * Redistributions in binary form must reproduce the above copyright notice, 
 *    this list of conditions and the following disclaimer in the documentation 
 *    and/or other materials provided with the distribution.
 * 
 *  * The names of its contributors may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 * POSSIBILITY OF SUCH DAMAGE.
 */

package net.byteseek.matcher.sequence;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Random;

import net.byteseek.bytes.ByteUtils;
import net.byteseek.io.reader.ByteArrayReader;
import net.byteseek.io.reader.FileReader;
import net.byteseek.matcher.bytes.ByteMatcher;
import net.byteseek.matcher.bytes.OneByteMatcher;
import net.byteseek.matcher.bytes.TwoByteMatcher;

import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * Tests all the constructors and public methods of the SequenceSequenceMatcher
 * class for both success and failure, and the associated
 * ReverseSequenceSequenceMatcher class for all the same conditions.
 * 
 * In particular, it tests for out-of-bounds, next-to-boundary and
 * boundary-crossing conditions in the matching methods.
 * 
 * @author <NAME>
 */
public class SequenceSequenceMatcherTest {

	// ////////////////
	// test setup    //
	// ////////////////

	private final static Random rand = new Random();

	private FileReader reader;
	private byte[] bytes;

	public SequenceSequenceMatcherTest() {
	}

	/**
	 * Generates a random number to use in randomising tests where complete
	 * coverage takes too long. The seed is output to the console to give a
	 * fighting chance of replicating a failing test - but I'm not really
	 * convinced this is a very pleasant way of testing. Still, it gives more
	 * complete coverage of the code than purely fixed tests.
	 * 
	 * @throws Exception
	 */
	@BeforeClass
	public static void setUpClass() throws Exception {
		final long seed = System.currentTimeMillis();
		// final long seed = ?
		rand.setSeed(seed);
		System.out.println("Seeding random number generator with: " + Long.toString(seed));
		System.out.println("To repeat these exact tests, set the seed to the value above.");
	}

	/**
	 * Creates a file reader and a byte array from an ASCII test file.
	 * 
	 * @throws Exception
	 */
	@Before
	public void setUp() throws Exception {
		reader = new FileReader(getFile("/TestASCII.txt"));
		bytes = reader.getWindow(0).getArray();
	}

	// /////////////////////////
	// constructor tests //
	// /////////////////////////

	/**
	 * Construct using random arrays of ByteMatchers, 100 times. Tests are:
	 * 
	 * - the length is correct. - each position in the matcher only matches one
	 * byte. - each byte in the matcher is correct. - that the array passed in
	 * was defensively copied.
	 */
	@Test
	public void testConstructByteMatcherArray() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final ByteMatcher[] array = createRandomByteMatcherArray(1024);
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(array);
			assertEquals("length:" + Integer.toString(array.length), array.length, matcher.length());

			for (int i = 0; i < array.length; i++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(i);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				final byte matchingValue = array[i].getMatchingBytes()[0];
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
			}

			testDefensivelyCopied(array, matcher);
		}
	}

	private void testDefensivelyCopied(ByteMatcher[] array, SequenceMatcher matcher) {
		changeArray(array);
		for (int pos = 0; pos < array.length; pos++) {
			final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[pos].getMatchingBytes()[0];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertTrue("byte value not equals:" + Integer.toString(matchingValue), matchingValue != matchingBytes[0]);
		}
	}

	private void changeArray(ByteMatcher[] array) {
		for (int i = 0; i < array.length; i++) {
			int newvalue = ((array[i].getMatchingBytes()[0] & 0xFF) + 1) % 256;
			array[i] = OneByteMatcher.valueOf((byte) newvalue);
		}
	}
	

	/**
	 * Construct using random repeated byte values for all byte values. Tests
	 * are:
	 * 
	 * - length is correct - each position in the matcher only matches one byte.
	 * - each byte in the matcher is correct.
	 * 
	 */
	@Test
	public void testConstructRepeatedBytes() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats,
					OneByteMatcher.valueOf((byte) byteValue));
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}
	

	@Test
	public void testRepeatedByteMatcher() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			ByteMatcher bytematcher = OneByteMatcher.valueOf((byte) byteValue);
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats, bytematcher);
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}


	@Test
	public void testConstructSingleByteMatcher() {
		for (int i = 0; i < 256; i++) {
			ByteMatcher matcher = OneByteMatcher.valueOf((byte) i);
			SequenceSequenceMatcher sequence = new SequenceSequenceMatcher(matcher);
			assertEquals("Length is one", 1, sequence.length());
			byte[] test = new byte[] { (byte) i };
			assertTrue("Matches byte value correctly " + i, sequence.matches(test, 0));
			int val2 = (i + 1) % 256;
			byte[] test2 = new byte[] { (byte) val2 };
			assertFalse("Does not match value " + val2, sequence.matches(test2, 0));
		}
	}

	
	@Test
	public void testConstructRepeatedByteMatcherArray() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats,
					OneByteMatcher.valueOf((byte) byteValue));
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}

	
	@Test
	public void testConstructRepeatedSequenceSequenceMatcher() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final ByteMatcher[] array = createRandomByteMatcherArray(1024);
			final SequenceSequenceMatcher source = new SequenceSequenceMatcher(array);
			final int repeats = rand.nextInt(10) + 1;
			final int length = array.length * repeats;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats, source);
			assertEquals("length:" + length, length, matcher.length());

			for (int pos = 0; pos < length; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				final int arrayPos = (pos % array.length);
				final byte matchingValue = array[arrayPos].getMatchingBytes()[0];
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
			}
		}
	}


	@Test
	public void testConstructSingleSequenceSequenceMatcher() {
		ByteMatcher[] array = createRandomByteMatcherArray(65);
		SequenceSequenceMatcher source = new SequenceSequenceMatcher(array);
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(source);
		assertEquals("Lengths are the same: " + array.length, source.length(), result.length());
		for (int i = 0; i < array.length; i++) {
			final ByteMatcher sbm = result.getMatcherForPosition(i);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[i].getMatchingBytes()[0];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
		}
	}

	
	@Test
	public void testConstructSequenceSequenceMatcherArray() {
		SequenceSequenceMatcher[] matchers = new SequenceSequenceMatcher[5];
		int totalLength = 0;
		for (int i = 0; i < 5; i++) {
			matchers[i] = new SequenceSequenceMatcher(createRandomByteMatcherArray(129));
			totalLength += matchers[i].length();
		}
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(matchers);
		assertEquals("Length is correct: " + totalLength, totalLength, result.length());

		int matcherPos = 0;
		for (int i = 0; i < 5; i++) {
			SequenceSequenceMatcher component = matchers[i];
			for (int j = 0; j < component.length(); j++) {
				ByteMatcher sourceMatch = component.getMatcherForPosition(j);
				ByteMatcher resultMatch = result.getMatcherForPosition(matcherPos++);
				byte[] sourceBytes = sourceMatch.getMatchingBytes();
				byte[] resultBytes = resultMatch.getMatchingBytes();
				assertEquals("Same number of bytes match", sourceBytes.length, resultBytes.length);
				assertEquals("Matches the same bytes", sourceBytes[0], resultBytes[0]);
			}
		}
	}

	@Test
	public void testConstructSingleSequenceMatcher() {
		byte[] array = createRandomArray(342);
		ByteSequenceMatcher source = new ByteSequenceMatcher(array);
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(source);
		assertEquals("length is correct " + array.length, source.length(), result.length());
		for (int i = 0; i < array.length; i++) {
			final ByteMatcher sbm = result.getMatcherForPosition(i);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[i];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
		}
	}

	@Test
	public void testConstructSequenceMatcherArray() {
		ByteSequenceMatcher[] matchers = new ByteSequenceMatcher[5];
		int totalLength = 0;
		for (int i = 0; i < 5; i++) {
			matchers[i] = new ByteSequenceMatcher(createRandomArray(129));
			totalLength += matchers[i].length();
		}
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(matchers);
		assertEquals("Length is correct: " + totalLength, totalLength, result.length());

		int matcherPos = 0;
		for (int i = 0; i < 5; i++) {
			ByteSequenceMatcher component = matchers[i];
			for (int j = 0; j < component.length(); j++) {
				ByteMatcher sourceMatch = component.getMatcherForPosition(j);
				ByteMatcher resultMatch = result.getMatcherForPosition(matcherPos++);
				byte[] sourceBytes = sourceMatch.getMatchingBytes();
				byte[] resultBytes = resultMatch.getMatchingBytes();
				assertEquals("Same number of bytes match", sourceBytes.length, resultBytes.length);
				assertEquals("Matches the same bytes", sourceBytes[0], resultBytes[0]);
			}
		}
	}

	/**
	 * Construct using random lists of byte sequence matchers. Tests are:
	 * 
	 * - the length of an assembled matcher is correct. - each position in the
	 * list of matchers matches only one byte. - each position in the assembled
	 * matcher matches only one byte. - each byte in the assembled matcher is
	 * correct.
	 */
	@Test
	public void testConstructByteSequenceMatcherList() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final List<SequenceSequenceMatcher> list = createRandomList(32);
			int totalLength = 0;
			for (final SequenceMatcher matcher : list) {
				totalLength += matcher.length();
			}
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(list);
			assertEquals("length:" + Integer.toString(totalLength), totalLength, matcher.length());

			int localPos = -1;
			int matchIndex = 0;
			SequenceMatcher currentMatcher = list.get(matchIndex);
			for (int pos = 0; pos < totalLength; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				localPos++;
				if (localPos == currentMatcher.length()) {
					matchIndex++;
					currentMatcher = list.get(matchIndex);
					localPos = 0;
				}
				final ByteMatcher sbm2 = currentMatcher.getMatcherForPosition(localPos);
				final byte[] matchingBytes2 = sbm2.getMatchingBytes();
				assertEquals("number of bytes matched source=1", 1, matchingBytes2.length);
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingBytes2[0]), matchingBytes2[0], matchingBytes[0]);
			}
		}
	}

	

	// ////////////////////////////////
	// construction failure tests //
	// ////////////////////////////////

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNoRepeats() {
		new SequenceSequenceMatcher(0, OneByteMatcher.valueOf((byte) 0x8f));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullArray() {
		new SequenceSequenceMatcher((SequenceMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyArray() {
		new SequenceSequenceMatcher(new SequenceMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullList() {
		new SequenceSequenceMatcher((ArrayList<SequenceSequenceMatcher>) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyList() {
		new SequenceSequenceMatcher(new ArrayList<SequenceSequenceMatcher>());
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullElementList() {
		List<SequenceSequenceMatcher> list = new ArrayList<SequenceSequenceMatcher>();
		list.add(null);
		new SequenceSequenceMatcher(list);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcherCollection() {
		new SequenceSequenceMatcher((List<ByteMatcher>) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyByteMatcherCollection() {
		new SequenceSequenceMatcher(new ArrayList<ByteMatcher>());
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructByteMatcherCollectionNullElement() {
		List<ByteMatcher> matchers = new ArrayList<ByteMatcher>();
		matchers.add(OneByteMatcher.valueOf((byte) 0x00));
		matchers.add(null);
		new SequenceSequenceMatcher(matchers);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullRepeatedByteMatcher() {
		new SequenceSequenceMatcher(3, (ByteMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteMatcher() {
		new SequenceSequenceMatcher(0, OneByteMatcher.valueOf((byte) 0x00));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNegativeRepeatedByteMatcher() {
		new SequenceSequenceMatcher(-1, OneByteMatcher.valueOf((byte) 0x00));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcher() {
		new SequenceSequenceMatcher((ByteMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcherArray() {
		new SequenceSequenceMatcher((ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullByteMatcherArray() {
		new SequenceSequenceMatcher(1, (ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteMatcherArray() {
		new SequenceSequenceMatcher(0, new ByteMatcher[] { OneByteMatcher.valueOf((byte) 0x00) });
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructByteMatcherArrayNullElement() {
		new SequenceSequenceMatcher(new ByteMatcher[] { null });
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyByteMatcherArray() {
		new SequenceSequenceMatcher(new ByteMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullByteMatcher() {
		new SequenceSequenceMatcher(5, (ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedEmptyByteArray() {
		new SequenceSequenceMatcher(5, new ByteMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteArray() {
		new SequenceSequenceMatcher(0, new ByteMatcher[1]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullSequenceSequenceMatcher() {
		new SequenceSequenceMatcher(1, (SequenceSequenceMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedSequenceSequenceMatcher() {
		new SequenceSequenceMatcher(0, new SequenceSequenceMatcher((byte) 0));
	}

	
	// ///////////////////////////
	// matcher iteration tests //
	// ///////////////////////////

	@Test
	public void testByteMatcherIterator() {
		testByteMatcherIterator("1");
		testByteMatcherIterator("abcdefghijklmnopqrstuvwxyz");
		testByteMatcherIterator("atat\t\tn\n");
	}

	private void testByteMatcherIterator(String value) {
		ByteSequenceMatcher source = new ByteSequenceMatcher(value);
		SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(source);
		testByteMatcherIterator(matcher, value);

		for (int i = 1; i < value.length() - 1; i++) {
			String newValue = value.substring(i);
			SequenceMatcher submatch = matcher.subsequence(i);
			testByteMatcherIterator(submatch, newValue);
		}

		for (int i = 1; i < value.length(); i++) {
			String newValue = value.substring(0, i);
			SequenceMatcher submatch = matcher.subsequence(0, i);
			testByteMatcherIterator(submatch, newValue);
		}
	}

	private void testByteMatcherIterator(SequenceMatcher value, String expected) {
		Iterator<ByteMatcher> iterator = value.iterator();
		assertNotNull("Byte matcher iterator is not null", iterator);

		try {
			iterator.remove();
			fail("expected an unsupported operation exception removing from the iterator");
		} catch (UnsupportedOperationException ex) {
		}
		;

		assertTrue("Byte matcher iterator must always have at least one element [" + expected + ']', iterator.hasNext());
		int position = 0;
		while (iterator.hasNext()) {
			ByteMatcher bm = iterator.next();
			byte[] matchingbytes = bm.getMatchingBytes();
			assertEquals("Matching bytes has one byte", 1, matchingbytes.length);

			byte b = (byte) expected.charAt(position++);
			assertEquals("Bytes match for byte " + b, b, matchingbytes[0]);
		}

		assertEquals("Lengths match after iteration", position, value.length());

		try {
			iterator.next();
			fail("expected NoSuchElementException");
		} catch (NoSuchElementException ex) {
		}
		;
	}

	private void testReverseByteMatcherIterator(SequenceMatcher value, String expected) {
		Iterator<ByteMatcher> iterator = value.iterator();
		assertNotNull("Byte matcher iterator is not null", iterator);

		try {
			iterator.remove();
			fail("expected an unsupported operation exception removing from the iterator");
		} catch (UnsupportedOperationException ex) {
		}
		;

		assertTrue("Byte matcher iterator must always have at least one element", iterator.hasNext());
		int position = expected.length();
		while (iterator.hasNext()) {
			ByteMatcher bm = iterator.next();
			byte[] matchingbytes = bm.getMatchingBytes();
			assertEquals("Matching bytes has one byte", 1, matchingbytes.length);

			byte b = (byte) expected.charAt(--position);
			assertEquals("Bytes match for byte " + b, b, matchingbytes[0]);
		}

		assertEquals("Lengths match after iteration", 0, position);

		try {
			iterator.next();
			fail("expected NoSuchElementException");
		} catch (NoSuchElementException ex) {
		}
		;
	}

	// /////////////////////////////
	// reader matching tests //
	// /////////////////////////////

	@Test
	public void testMatches_ByteReader_long() throws FileNotFoundException, IOException {
		SequenceMatcher matcher = new SequenceSequenceMatcher(3, OneByteMatcher.valueOf((byte) 0x2A));
		runTestMatchesAround(matcher, 0, 61, 1017);

		matcher = new SequenceSequenceMatcher(3, OneByteMatcher.valueOf((byte) 0x2A)).reverse();
		runTestMatchesAround(matcher, 0, 61, 1017);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher("Here"));
		runTestMatchesAround(matcher, 28200, 60836, 64481);

		matcher = matcher.subsequence(1, 4);
		runTestMatchesAround(matcher, 28201, 60837, 64482);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher("ereH")).reverse();
		runTestMatchesAround(matcher, 28200, 60836, 64481);

		matcher = matcher.subsequence(1, 3);
		runTestMatchesAround(matcher, 28201, 60837, 64482);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher(new byte[] { 0x2e, 0x0d, 0x0a }));
		runTestMatchesAround(matcher, 196, 42004, 112277);

		matcher = matcher.subsequence(1);
		runTestMatchesAround(matcher, 197, 42005, 112278);
<commit_msg>Fix minor warnings.<commit_after>/*
 * Copyright <NAME> 2014, All rights reserved.
 *
 * This code is licensed under a standard 3-clause BSD license:
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice, 
 *    this list of conditions and the following disclaimer.
 * 
 *  * Redistributions in binary form must reproduce the above copyright notice, 
 *    this list of conditions and the following disclaimer in the documentation 
 *    and/or other materials provided with the distribution.
 * 
 *  * The names of its contributors may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 * POSSIBILITY OF SUCH DAMAGE.
 */

package net.byteseek.matcher.sequence;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Random;

import net.byteseek.io.reader.FileReader;
import net.byteseek.matcher.bytes.ByteMatcher;
import net.byteseek.matcher.bytes.OneByteMatcher;
import net.byteseek.matcher.bytes.TwoByteMatcher;

import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * Tests all the constructors and public methods of the SequenceSequenceMatcher
 * class for both success and failure, and the associated
 * ReverseSequenceSequenceMatcher class for all the same conditions.
 * 
 * In particular, it tests for out-of-bounds, next-to-boundary and
 * boundary-crossing conditions in the matching methods.
 * 
 * @author <NAME>
 */
public class SequenceSequenceMatcherTest {

	// ////////////////
	// test setup    //
	// ////////////////

	private final static Random rand = new Random();

	private FileReader reader;
	private byte[] bytes;

	public SequenceSequenceMatcherTest() {
	}

	/**
	 * Generates a random number to use in randomising tests where complete
	 * coverage takes too long. The seed is output to the console to give a
	 * fighting chance of replicating a failing test - but I'm not really
	 * convinced this is a very pleasant way of testing. Still, it gives more
	 * complete coverage of the code than purely fixed tests.
	 * 
	 * @throws Exception
	 */
	@BeforeClass
	public static void setUpClass() throws Exception {
		final long seed = System.currentTimeMillis();
		// final long seed = ?
		rand.setSeed(seed);
		System.out.println("Seeding random number generator with: " + Long.toString(seed));
		System.out.println("To repeat these exact tests, set the seed to the value above.");
	}

	/**
	 * Creates a file reader and a byte array from an ASCII test file.
	 * 
	 * @throws Exception
	 */
	@Before
	public void setUp() throws Exception {
		reader = new FileReader(getFile("/TestASCII.txt"));
		bytes = reader.getWindow(0).getArray();
	}

	// /////////////////////////
	// constructor tests //
	// /////////////////////////

	/**
	 * Construct using random arrays of ByteMatchers, 100 times. Tests are:
	 * 
	 * - the length is correct. - each position in the matcher only matches one
	 * byte. - each byte in the matcher is correct. - that the array passed in
	 * was defensively copied.
	 */
	@Test
	public void testConstructByteMatcherArray() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final ByteMatcher[] array = createRandomByteMatcherArray(1024);
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(array);
			assertEquals("length:" + Integer.toString(array.length), array.length, matcher.length());

			for (int i = 0; i < array.length; i++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(i);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				final byte matchingValue = array[i].getMatchingBytes()[0];
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
			}

			testDefensivelyCopied(array, matcher);
		}
	}

	private void testDefensivelyCopied(ByteMatcher[] array, SequenceMatcher matcher) {
		changeArray(array);
		for (int pos = 0; pos < array.length; pos++) {
			final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[pos].getMatchingBytes()[0];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertTrue("byte value not equals:" + Integer.toString(matchingValue), matchingValue != matchingBytes[0]);
		}
	}

	private void changeArray(ByteMatcher[] array) {
		for (int i = 0; i < array.length; i++) {
			int newvalue = ((array[i].getMatchingBytes()[0] & 0xFF) + 1) % 256;
			array[i] = OneByteMatcher.valueOf((byte) newvalue);
		}
	}
	

	/**
	 * Construct using random repeated byte values for all byte values. Tests
	 * are:
	 * 
	 * - length is correct - each position in the matcher only matches one byte.
	 * - each byte in the matcher is correct.
	 * 
	 */
	@Test
	public void testConstructRepeatedBytes() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats,
					OneByteMatcher.valueOf((byte) byteValue));
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}
	

	@Test
	public void testRepeatedByteMatcher() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			ByteMatcher bytematcher = OneByteMatcher.valueOf((byte) byteValue);
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats, bytematcher);
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}


	@Test
	public void testConstructSingleByteMatcher() {
		for (int i = 0; i < 256; i++) {
			ByteMatcher matcher = OneByteMatcher.valueOf((byte) i);
			SequenceSequenceMatcher sequence = new SequenceSequenceMatcher(matcher);
			assertEquals("Length is one", 1, sequence.length());
			byte[] test = new byte[] { (byte) i };
			assertTrue("Matches byte value correctly " + i, sequence.matches(test, 0));
			int val2 = (i + 1) % 256;
			byte[] test2 = new byte[] { (byte) val2 };
			assertFalse("Does not match value " + val2, sequence.matches(test2, 0));
		}
	}

	
	@Test
	public void testConstructRepeatedByteMatcherArray() {
		for (int byteValue = 0; byteValue < 256; byteValue++) {
			final int repeats = rand.nextInt(1024) + 1;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats,
					OneByteMatcher.valueOf((byte) byteValue));
			assertEquals("length:" + Integer.toString(repeats) + ", byte value:" + Integer.toString(byteValue),
					repeats, matcher.length());

			for (int pos = 0; pos < repeats; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(byteValue), byteValue, matchingBytes[0] & 0xFF);
			}
		}
	}

	
	@Test
	public void testConstructRepeatedSequenceSequenceMatcher() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final ByteMatcher[] array = createRandomByteMatcherArray(1024);
			final SequenceSequenceMatcher source = new SequenceSequenceMatcher(array);
			final int repeats = rand.nextInt(10) + 1;
			final int length = array.length * repeats;
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(repeats, source);
			assertEquals("length:" + length, length, matcher.length());

			for (int pos = 0; pos < length; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				final int arrayPos = (pos % array.length);
				final byte matchingValue = array[arrayPos].getMatchingBytes()[0];
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
			}
		}
	}


	@Test
	public void testConstructSingleSequenceSequenceMatcher() {
		ByteMatcher[] array = createRandomByteMatcherArray(65);
		SequenceSequenceMatcher source = new SequenceSequenceMatcher(array);
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(source);
		assertEquals("Lengths are the same: " + array.length, source.length(), result.length());
		for (int i = 0; i < array.length; i++) {
			final ByteMatcher sbm = result.getMatcherForPosition(i);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[i].getMatchingBytes()[0];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
		}
	}

	
	@Test
	public void testConstructSequenceSequenceMatcherArray() {
		SequenceSequenceMatcher[] matchers = new SequenceSequenceMatcher[5];
		int totalLength = 0;
		for (int i = 0; i < 5; i++) {
			matchers[i] = new SequenceSequenceMatcher(createRandomByteMatcherArray(129));
			totalLength += matchers[i].length();
		}
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(matchers);
		assertEquals("Length is correct: " + totalLength, totalLength, result.length());

		int matcherPos = 0;
		for (int i = 0; i < 5; i++) {
			SequenceSequenceMatcher component = matchers[i];
			for (int j = 0; j < component.length(); j++) {
				ByteMatcher sourceMatch = component.getMatcherForPosition(j);
				ByteMatcher resultMatch = result.getMatcherForPosition(matcherPos++);
				byte[] sourceBytes = sourceMatch.getMatchingBytes();
				byte[] resultBytes = resultMatch.getMatchingBytes();
				assertEquals("Same number of bytes match", sourceBytes.length, resultBytes.length);
				assertEquals("Matches the same bytes", sourceBytes[0], resultBytes[0]);
			}
		}
	}

	@Test
	public void testConstructSingleSequenceMatcher() {
		byte[] array = createRandomArray(342);
		ByteSequenceMatcher source = new ByteSequenceMatcher(array);
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(source);
		assertEquals("length is correct " + array.length, source.length(), result.length());
		for (int i = 0; i < array.length; i++) {
			final ByteMatcher sbm = result.getMatcherForPosition(i);
			final byte[] matchingBytes = sbm.getMatchingBytes();
			final byte matchingValue = array[i];
			assertEquals("number of bytes matched=1", 1, matchingBytes.length);
			assertEquals("byte value:" + Integer.toString(matchingValue), matchingValue, matchingBytes[0]);
		}
	}

	@Test
	public void testConstructSequenceMatcherArray() {
		ByteSequenceMatcher[] matchers = new ByteSequenceMatcher[5];
		int totalLength = 0;
		for (int i = 0; i < 5; i++) {
			matchers[i] = new ByteSequenceMatcher(createRandomArray(129));
			totalLength += matchers[i].length();
		}
		SequenceSequenceMatcher result = new SequenceSequenceMatcher(matchers);
		assertEquals("Length is correct: " + totalLength, totalLength, result.length());

		int matcherPos = 0;
		for (int i = 0; i < 5; i++) {
			ByteSequenceMatcher component = matchers[i];
			for (int j = 0; j < component.length(); j++) {
				ByteMatcher sourceMatch = component.getMatcherForPosition(j);
				ByteMatcher resultMatch = result.getMatcherForPosition(matcherPos++);
				byte[] sourceBytes = sourceMatch.getMatchingBytes();
				byte[] resultBytes = resultMatch.getMatchingBytes();
				assertEquals("Same number of bytes match", sourceBytes.length, resultBytes.length);
				assertEquals("Matches the same bytes", sourceBytes[0], resultBytes[0]);
			}
		}
	}

	/**
	 * Construct using random lists of byte sequence matchers. Tests are:
	 * 
	 * - the length of an assembled matcher is correct. - each position in the
	 * list of matchers matches only one byte. - each position in the assembled
	 * matcher matches only one byte. - each byte in the assembled matcher is
	 * correct.
	 */
	@Test
	public void testConstructByteSequenceMatcherList() {
		for (int testNo = 0; testNo < 10; testNo++) {
			final List<SequenceSequenceMatcher> list = createRandomList(32);
			int totalLength = 0;
			for (final SequenceMatcher matcher : list) {
				totalLength += matcher.length();
			}
			final SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(list);
			assertEquals("length:" + Integer.toString(totalLength), totalLength, matcher.length());

			int localPos = -1;
			int matchIndex = 0;
			SequenceMatcher currentMatcher = list.get(matchIndex);
			for (int pos = 0; pos < totalLength; pos++) {
				final ByteMatcher sbm = matcher.getMatcherForPosition(pos);
				final byte[] matchingBytes = sbm.getMatchingBytes();
				localPos++;
				if (localPos == currentMatcher.length()) {
					matchIndex++;
					currentMatcher = list.get(matchIndex);
					localPos = 0;
				}
				final ByteMatcher sbm2 = currentMatcher.getMatcherForPosition(localPos);
				final byte[] matchingBytes2 = sbm2.getMatchingBytes();
				assertEquals("number of bytes matched source=1", 1, matchingBytes2.length);
				assertEquals("number of bytes matched=1", 1, matchingBytes.length);
				assertEquals("byte value:" + Integer.toString(matchingBytes2[0]), matchingBytes2[0], matchingBytes[0]);
			}
		}
	}

	

	// ////////////////////////////////
	// construction failure tests //
	// ////////////////////////////////

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNoRepeats() {
		new SequenceSequenceMatcher(0, OneByteMatcher.valueOf((byte) 0x8f));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullArray() {
		new SequenceSequenceMatcher((SequenceMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyArray() {
		new SequenceSequenceMatcher(new SequenceMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullList() {
		new SequenceSequenceMatcher((ArrayList<SequenceSequenceMatcher>) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyList() {
		new SequenceSequenceMatcher(new ArrayList<SequenceSequenceMatcher>());
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullElementList() {
		List<SequenceSequenceMatcher> list = new ArrayList<SequenceSequenceMatcher>();
		list.add(null);
		new SequenceSequenceMatcher(list);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcherCollection() {
		new SequenceSequenceMatcher((List<ByteMatcher>) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyByteMatcherCollection() {
		new SequenceSequenceMatcher(new ArrayList<ByteMatcher>());
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructByteMatcherCollectionNullElement() {
		List<ByteMatcher> matchers = new ArrayList<ByteMatcher>();
		matchers.add(OneByteMatcher.valueOf((byte) 0x00));
		matchers.add(null);
		new SequenceSequenceMatcher(matchers);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullRepeatedByteMatcher() {
		new SequenceSequenceMatcher(3, (ByteMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteMatcher() {
		new SequenceSequenceMatcher(0, OneByteMatcher.valueOf((byte) 0x00));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNegativeRepeatedByteMatcher() {
		new SequenceSequenceMatcher(-1, OneByteMatcher.valueOf((byte) 0x00));
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcher() {
		new SequenceSequenceMatcher((ByteMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructNullByteMatcherArray() {
		new SequenceSequenceMatcher((ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullByteMatcherArray() {
		new SequenceSequenceMatcher(1, (ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteMatcherArray() {
		new SequenceSequenceMatcher(0, new ByteMatcher[] { OneByteMatcher.valueOf((byte) 0x00) });
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructByteMatcherArrayNullElement() {
		new SequenceSequenceMatcher(new ByteMatcher[] { null });
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructEmptyByteMatcherArray() {
		new SequenceSequenceMatcher(new ByteMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullByteMatcher() {
		new SequenceSequenceMatcher(5, (ByteMatcher[]) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedEmptyByteArray() {
		new SequenceSequenceMatcher(5, new ByteMatcher[0]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedByteArray() {
		new SequenceSequenceMatcher(0, new ByteMatcher[1]);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructRepeatedNullSequenceSequenceMatcher() {
		new SequenceSequenceMatcher(1, (SequenceSequenceMatcher) null);
	}

	@SuppressWarnings("unused")
	@Test(expected = IllegalArgumentException.class)
	public void testConstructZeroRepeatedSequenceSequenceMatcher() {
		new SequenceSequenceMatcher(0, new SequenceSequenceMatcher((byte) 0));
	}

	
	// ///////////////////////////
	// matcher iteration tests //
	// ///////////////////////////

	@Test
	public void testByteMatcherIterator() {
		testByteMatcherIterator("1");
		testByteMatcherIterator("abcdefghijklmnopqrstuvwxyz");
		testByteMatcherIterator("atat\t\tn\n");
	}

	private void testByteMatcherIterator(String value) {
		ByteSequenceMatcher source = new ByteSequenceMatcher(value);
		SequenceSequenceMatcher matcher = new SequenceSequenceMatcher(source);
		testByteMatcherIterator(matcher, value);

		for (int i = 1; i < value.length() - 1; i++) {
			String newValue = value.substring(i);
			SequenceMatcher submatch = matcher.subsequence(i);
			testByteMatcherIterator(submatch, newValue);
		}

		for (int i = 1; i < value.length(); i++) {
			String newValue = value.substring(0, i);
			SequenceMatcher submatch = matcher.subsequence(0, i);
			testByteMatcherIterator(submatch, newValue);
		}
	}

	private void testByteMatcherIterator(SequenceMatcher value, String expected) {
		Iterator<ByteMatcher> iterator = value.iterator();
		assertNotNull("Byte matcher iterator is not null", iterator);

		try {
			iterator.remove();
			fail("expected an unsupported operation exception removing from the iterator");
		} catch (UnsupportedOperationException ex) {
		}
		;

		assertTrue("Byte matcher iterator must always have at least one element [" + expected + ']', iterator.hasNext());
		int position = 0;
		while (iterator.hasNext()) {
			ByteMatcher bm = iterator.next();
			byte[] matchingbytes = bm.getMatchingBytes();
			assertEquals("Matching bytes has one byte", 1, matchingbytes.length);

			byte b = (byte) expected.charAt(position++);
			assertEquals("Bytes match for byte " + b, b, matchingbytes[0]);
		}

		assertEquals("Lengths match after iteration", position, value.length());

		try {
			iterator.next();
			fail("expected NoSuchElementException");
		} catch (NoSuchElementException ex) {
		}
		;
	}


	// /////////////////////////////
	// reader matching tests //
	// /////////////////////////////

	@Test
	public void testMatches_ByteReader_long() throws FileNotFoundException, IOException {
		SequenceMatcher matcher = new SequenceSequenceMatcher(3, OneByteMatcher.valueOf((byte) 0x2A));
		runTestMatchesAround(matcher, 0, 61, 1017);

		matcher = new SequenceSequenceMatcher(3, OneByteMatcher.valueOf((byte) 0x2A)).reverse();
		runTestMatchesAround(matcher, 0, 61, 1017);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher("Here"));
		runTestMatchesAround(matcher, 28200, 60836, 64481);

		matcher = matcher.subsequence(1, 4);
		runTestMatchesAround(matcher, 28201, 60837, 64482);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher("ereH")).reverse();
		runTestMatchesAround(matcher, 28200, 60836, 64481);

		matcher = matcher.subsequence(1, 3);
		runTestMatchesAround(matcher, 28201, 60837, 64482);

		matcher = new SequenceSequenceMatcher(new ByteSequenceMatcher(new byte[] { 0x2e, 0x0d, 0x0a }));
		runTestMatchesAround(matcher, 196, 42004, 112277);

		matcher = matcher.subsequence(1);
		runTestMatchesAround(matcher, 197, 42005, 112278);

{'fraction_non_alphanumeric': 0.1208328951519613, 'fraction_numerical': 0.013818487748448838, 'mean_word_length': 4.68732057416268, 'pattern_counts': {'":': 0, '<': 78, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6440547', 'n_tokens_mistral': 14910, 'n_tokens_neox': 13092, 'n_words': 3687}