<commit_before>package ar.glyphsets;

import java.awt.geom.Rectangle2D;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.Iterator;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveTask;

import ar.Glyph;
import ar.Glyphset;
import ar.glyphsets.implicitgeometry.Indexed;
import ar.glyphsets.implicitgeometry.IndexedEncoding;
import ar.glyphsets.implicitgeometry.Shaper;
import ar.glyphsets.implicitgeometry.Valuer;
import ar.util.memoryMapping.MappedFile;
import ar.util.memoryMapping.MemMapEncoder;
import ar.util.memoryMapping.MemMapEncoder.TYPE;
import ar.util.Util;

/**Implicit geometry, sequentially arranged glyphset backed by a memory-mapped file.
 * 
 * This glyphset uses 'implicit geometry' in that the geometry is produced just-in-time and
 * discarded immediately.  Implicit geometry significantly reduces the required memory at the
 * cost of speed.  When using implicit geometry, the display window size is the principal 
 * memory consumer (because it determines both the image size and the aggregates set size). 
 * 
 * The memory mapped file must be encoded as fixed-width records for this class.
 * The files may include a header to self-describe or the header information may be supplied.
 * 
 *  The header, when provided, is an integer indicating how many fields are in each record,
 *  followed by a set of characters (one for each field).  
 *  
 *  This is class is NOT thread-safe.  However, subsets are logically independent units 
 *  so multiple subsets can be safely created and used concurrently (even if they overlap).
 *  
 *  The characters that describe field types are:
 *  
 *   +   s -- Short (two bytes)
 *   +   i -- Int (four bytes)
 *   +   l -- Long (eight bytes)
 *   +   f -- Float (four bytes)
 *   +   d -- Double (eight bytes)
 *   +   c -- Char (two bytes)
 *   +   b -- Byte (one byte)
 *   
 * @author jcottam
 *
 */
public class MemMapList<G,I> implements Glyphset.RandomAccess<G,I> {
	/**Flag field indicating the binary file encoding (hbin) version understood by the parser.**/
	public static final int VERSION_UNDERSTOOD = -1;
	
	/**How large should backing read buffer be?
	 * Reducing this number tends to result in faster thread startup times, but slower overall run-times.
	 * **/
	public static int BUFFER_BYTES = Integer.MAX_VALUE;
	
	/**Thread-pool size for parallel operations.**/
	public static int THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
	private final ForkJoinPool pool = new ForkJoinPool(THREAD_POOL_SIZE);

	private final MappedFile buffer;

	private final TYPE[] types;
	private final Valuer<Indexed,I> valuer;
	private final Shaper<G,Indexed> shaper;<commit_msg>All MemMapLists share the same thread pool for parallel bounds calculations.<commit_after>package ar.glyphsets;

import java.awt.geom.Rectangle2D;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.Iterator;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveTask;

import ar.Glyph;
import ar.Glyphset;
import ar.glyphsets.implicitgeometry.Indexed;
import ar.glyphsets.implicitgeometry.IndexedEncoding;
import ar.glyphsets.implicitgeometry.Shaper;
import ar.glyphsets.implicitgeometry.Valuer;
import ar.util.memoryMapping.MappedFile;
import ar.util.memoryMapping.MemMapEncoder;
import ar.util.memoryMapping.MemMapEncoder.TYPE;
import ar.util.Util;

/**Implicit geometry, sequentially arranged glyphset backed by a memory-mapped file.
 * 
 * This glyphset uses 'implicit geometry' in that the geometry is produced just-in-time and
 * discarded immediately.  Implicit geometry significantly reduces the required memory at the
 * cost of speed.  When using implicit geometry, the display window size is the principal 
 * memory consumer (because it determines both the image size and the aggregates set size). 
 * 
 * The memory mapped file must be encoded as fixed-width records for this class.
 * The files may include a header to self-describe or the header information may be supplied.
 * 
 *  The header, when provided, is an integer indicating how many fields are in each record,
 *  followed by a set of characters (one for each field).  
 *  
 *  This is class is NOT thread-safe.  However, subsets are logically independent units 
 *  so multiple subsets can be safely created and used concurrently (even if they overlap).
 *  
 *  The characters that describe field types are:
 *  
 *   +   s -- Short (two bytes)
 *   +   i -- Int (four bytes)
 *   +   l -- Long (eight bytes)
 *   +   f -- Float (four bytes)
 *   +   d -- Double (eight bytes)
 *   +   c -- Char (two bytes)
 *   +   b -- Byte (one byte)
 *   
 * @author jcottam
 *
 */
public class MemMapList<G,I> implements Glyphset.RandomAccess<G,I> {
	/**Flag field indicating the binary file encoding (hbin) version understood by the parser.**/
	public static final int VERSION_UNDERSTOOD = -1;
	
	/**How large should backing read buffer be?
	 * Reducing this number tends to result in faster thread startup times, but slower overall run-times.
	 * **/
	public static int BUFFER_BYTES = Integer.MAX_VALUE;
	
	/**Thread-pool size for parallel operations.**/
	private static final ForkJoinPool pool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());

	private final MappedFile buffer;

	private final TYPE[] types;
	private final Valuer<Indexed,I> valuer;
	private final Shaper<G,Indexed> shaper;
{'fraction_non_alphanumeric': 0.08394027676620539, 'fraction_numerical': 0.0007283321194464676, 'mean_word_length': 4.03021978021978, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6423949', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1530, 'n_words': 599}