<commit_before>package org.dap.data_structures;

import java.io.Serializable;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

import org.dap.common.DapAPI;
import org.dap.common.DapException;
import org.dap.index.Index;

/**
 * Document is a data-structure that contains:
 * <ol>
 * <li>text</li>
 * <li>annotations</li>
 * <li>features</li>
 * </ol>
 * 
 * The most important functionality of {@link Document}s is to iterate over annotations in the document. For this, several
 * <code>iterator()</code> and <code>iterable()</code> methods are provided, which let the user to iterate over all the annotations,
 * or some of them.
 *
 * <p>
 * Date: 30 May 2017
 * @author <NAME>
 *
 */
@DapAPI
public final class Document implements Serializable, Iterable<Annotation<?>>
{
	private static final long serialVersionUID = -6556213855743551941L;
	
	/**
	 * Constructor with the document-name and the document-text.
	 * <br>
	 * Note that a default {@link DocumentCollection} will be created for this document.
	 * @param name document-name
	 * @param text document-text
	 */
	@DapAPI
	public Document(String name, String text)
	{
		this(name, new DocumentCollection(), text);
	}
	
	/**
	 * Constructor with the document-name, the {@link DocumentCollection} into which this document will be inserted,
	 * and the document-text.
	 * @param name document-name
	 * @param documentCollection the {@link DocumentCollection} into which this document will be inserted<commit_msg>Add comment<commit_after>package org.dap.data_structures;

import java.io.Serializable;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

import org.dap.common.DapAPI;
import org.dap.common.DapException;
import org.dap.index.Index;

/**
 * Document is a data-structure that contains:
 * <ol>
 * <li>text</li>
 * <li>annotations</li>
 * <li>features</li>
 * </ol>
 * 
 * The most important functionality of {@link Document}s is to iterate over annotations in the document. For this, several
 * <code>iterator()</code> and <code>iterable()</code> methods are provided, which let the user to iterate over all the annotations,
 * or some of them.
 * 
 * <p>
 * <b>Documents must not change when iterating over annotations.</b>
 * Adding or removing annotations while iterating over the document's annotations is a logical error.
 * The result of such an operation is undefined.
 * It might result in some unexplained exception, or, even worse, wrong annotations returned from the iterator.
 * See the tutorial for more information.
 *
 * <p>
 * Date: 30 May 2017
 * @author <NAME>
 *
 */
@DapAPI
public final class Document implements Serializable, Iterable<Annotation<?>>
{
	private static final long serialVersionUID = -6556213855743551941L;
	
	/**
	 * Constructor with the document-name and the document-text.
	 * <br>
	 * Note that a default {@link DocumentCollection} will be created for this document.
	 * @param name document-name
	 * @param text document-text
	 */
	@DapAPI
	public Document(String name, String text)
	{
		this(name, new DocumentCollection(), text);
	}
	
	/**
	 * Constructor with the document-name, the {@link DocumentCollection} into which this document will be inserted,
	 * and the document-text.
	 * @param name document-name
	 * @param documentCollection the {@link DocumentCollection} into which this document will be inserted
{'fraction_non_alphanumeric': 0.10590228748940977, 'fraction_numerical': 0.01412030499858797, 'mean_word_length': 4.350453172205438, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3309082', 'n_tokens_mistral': 1080, 'n_tokens_neox': 996, 'n_words': 369}