// Path: src/main/java/com/asher_stern/crf/utilities/TaggedToken.java
// public class TaggedToken<K,G>
// {
// 	public TaggedToken(K token, G tag)
// 	{
// 		super();
// 		this.token = token;
// 		this.tag = tag;
// 	}
// 	
// 	
// 	
// 	public K getToken()
// 	{
// 		return token;
// 	}
// 	public G getTag()
// 	{
// 		return tag;
// 	}
// 
// 	
// 
// 
// 	@Override
// 	public String toString()
// 	{
// 		return "CrfTaggedToken [getToken()=" + getToken() + ", getTag()="
// 				+ getTag() + "]";
// 	}
// 	
// 	
// 
// 
// 
// 
// 	@Override
// 	public int hashCode()
// 	{
// 		final int prime = 31;
// 		int result = 1;
// 		result = prime * result + ((tag == null) ? 0 : tag.hashCode());
// 		result = prime * result + ((token == null) ? 0 : token.hashCode());
// 		return result;
// 	}
// 
// 
// 
// 	@Override
// 	public boolean equals(Object obj)
// 	{
// 		if (this == obj)
// 			return true;
// 		if (obj == null)
// 			return false;
// 		if (getClass() != obj.getClass())
// 			return false;
// 		TaggedToken<?,?> other = (TaggedToken<?,?>) obj;
// 		if (tag == null)
// 		{
// 			if (other.tag != null)
// 				return false;
// 		} else if (!tag.equals(other.tag))
// 			return false;
// 		if (token == null)
// 		{
// 			if (other.token != null)
// 				return false;
// 		} else if (!token.equals(other.token))
// 			return false;
// 		return true;
// 	}
// 
// 
// 
// 
// 
// 
// 	private final K token;
// 	private final G tag;
// }


import java.util.Iterator;
import java.util.List;
import com.asher_stern.crf.utilities.TaggedToken;
package com.asher_stern.crf.postagging.data;



/**
 * Splits a given corpus into a train corpus and a test corpus.
 * <P>
 * If train-size is provided, and is larger than zero, then the first "train-size" sentences are returned as the train corpus.
 * Otherwise, the whole corpus is provided as the train corpus.
 * <P>
 * If the "train-size" is provided, then the test corpus contains <B>only</B> sentences that are not included in the train corpus.
 * If "test-size" is provided, then only the first "test-size" sentences are included in the test corpus, where "the first sentences"
 * means those which immediately the last sentence in the training (if "train-size" > 0), or the first sentences in the original
 * corpus (if "train-size<=0).
 *  
 * 
 * @author Asher Stern
 * Date: Nov 5, 2014
 *
 */
public class TrainTestPosTagCorpus<K,G>
{
// Path: src/main/java/com/asher_stern/crf/utilities/TaggedToken.java
// public class TaggedToken<K,G>
// {
// 	public TaggedToken(K token, G tag)
// 	{
// 		super();
// 		this.token = token;
// 		this.tag = tag;
// 	}
// 	
// 	
// 	
// 	public K getToken()
// 	{
// 		return token;
// 	}
// 	public G getTag()
// 	{
// 		return tag;
// 	}
// 
// 	
// 
// 
// 	@Override
// 	public String toString()
// 	{
// 		return "CrfTaggedToken [getToken()=" + getToken() + ", getTag()="
// 				+ getTag() + "]";
// 	}
// 	
// 	
// 
// 
// 
// 
// 	@Override
// 	public int hashCode()
// 	{
// 		final int prime = 31;
// 		int result = 1;
// 		result = prime * result + ((tag == null) ? 0 : tag.hashCode());
// 		result = prime * result + ((token == null) ? 0 : token.hashCode());
// 		return result;
// 	}
// 
// 
// 
// 	@Override
// 	public boolean equals(Object obj)
// 	{
// 		if (this == obj)
// 			return true;
// 		if (obj == null)
// 			return false;
// 		if (getClass() != obj.getClass())
// 			return false;
// 		TaggedToken<?,?> other = (TaggedToken<?,?>) obj;
// 		if (tag == null)
// 		{
// 			if (other.tag != null)
// 				return false;
// 		} else if (!tag.equals(other.tag))
// 			return false;
// 		if (token == null)
// 		{
// 			if (other.token != null)
// 				return false;
// 		} else if (!token.equals(other.token))
// 			return false;
// 		return true;
// 	}
// 
// 
// 
// 
// 
// 
// 	private final K token;
// 	private final G tag;
// }


// Path: src/main/java/com/asher_stern/crf/postagging/data/TrainTestPosTagCorpus.java
import java.util.Iterator;
import java.util.List;
import com.asher_stern.crf.utilities.TaggedToken;

package com.asher_stern.crf.postagging.data;



/**
 * Splits a given corpus into a train corpus and a test corpus.
 * <P>
 * If train-size is provided, and is larger than zero, then the first "train-size" sentences are returned as the train corpus.
 * Otherwise, the whole corpus is provided as the train corpus.
 * <P>
 * If the "train-size" is provided, then the test corpus contains <B>only</B> sentences that are not included in the train corpus.
 * If "test-size" is provided, then only the first "test-size" sentences are included in the test corpus, where "the first sentences"
 * means those which immediately the last sentence in the training (if "train-size" > 0), or the first sentences in the original
 * corpus (if "train-size<=0).
 *  
 * 
 * @author Asher Stern
 * Date: Nov 5, 2014
 *
 */
public class TrainTestPosTagCorpus<K,G>
{
	public TrainTestPosTagCorpus(int trainSize, Iterable<List<TaggedToken<K, G>>> realCorpus)