<commit_before>package com.akiba.cserver.store;

import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import junit.framework.TestCase;

import com.akiba.cserver.CServerConstants;
import com.akiba.cserver.FieldDef;
import com.akiba.cserver.FieldType;
import com.akiba.cserver.RowData;
import com.akiba.cserver.RowDef;
import com.akiba.cserver.RowDefCache;
import com.akiba.cserver.CServerUtil;
import com.persistit.Key;
import com.persistit.Persistit;

public class PersistitStoreTest extends TestCase implements CServerConstants {

	private final static File DATA_PATH = new File("/tmp/data");

	private final static RowDef ROW_DEF0 = RowDef.createRowDef(1234,
			new FieldDef[] { new FieldDef("t1", FieldType.INT),
					new FieldDef("t2", FieldType.INT),
					new FieldDef("t3", FieldType.INT) }, "test",
			"test_group_table", new int[] { 0 });

	//
	// Highly abbreviate "COI" structure.
	//
	private final static RowDef ROW_DEF_C = RowDef.createRowDef(100,
			new FieldDef[] { new FieldDef("customer_id", FieldType.INT),
					new FieldDef("customer_x", FieldType.VARCHAR, 100) },
			"customers", "coi", new int[] { 0 });

	private final static RowDef ROW_DEF_O = RowDef.createRowDef(101,
			new FieldDef[] { new FieldDef("order_id", FieldType.INT),
					new FieldDef("date", FieldType.INT),
					new FieldDef("description", FieldType.VARCHAR, 100) },
			"orders", "coi", new int[] { 0 }, 100, new int[] { 1 });

	private final static RowDef ROW_DEF_I = RowDef.createRowDef(102,
			new FieldDef[] { new FieldDef("item_id", FieldType.INT),
					new FieldDef("qty", FieldType.INT),
					new FieldDef("description", FieldType.VARCHAR, 100) },
			"items", "coi", new int[] { 0 }, 101, new int[] { 1 });

	private PersistitStore store;

	private RowDefCache rowDefCache;

	@Override
	public void setUp() throws Exception {
		rowDefCache = new RowDefCache();
		store = new PersistitStore(rowDefCache);
		CServerUtil.cleanUpDirectory(DATA_PATH);
		PersistitStore.setDataPath(DATA_PATH.getPath());
		store.startUp();
	}

	@Override
	public void tearDown() throws Exception {
		store.shutDown();
		store = null;
		rowDefCache = null;
	}

	public void testWriteSimpleRootRow() throws Exception {
		rowDefCache.putRowDef(ROW_DEF0);
		final RowData rowData = new RowData(new byte[64]);
		rowData.createRow(ROW_DEF0, new Object[] { 1, 2, 3 });
		assertEquals(OK, store.writeRow(rowData));
	}

	public void testWriteCOIrows() throws Exception {
		rowDefCache.putRowDef(ROW_DEF_C);
		rowDefCache.putRowDef(ROW_DEF_O);
		rowDefCache.putRowDef(ROW_DEF_I);
		final RowData rowC = new RowData(new byte[64]);
		final RowData rowO = new RowData(new byte[64]);
		final RowData rowI = new RowData(new byte[64]);

		for (int c = 0; ++c <= 10;) {
			int cid = c;
			rowC.reset(0, 64);
			rowC.createRow(ROW_DEF_C, new Object[] { cid, "Customer_" + cid });
			assertEquals(OK, store.writeRow(rowC));
			for (int o = 0; ++o <= 10;) {
				int oid = cid * 1000 + o;
				rowO.reset(0, 64);
				rowO.createRow(ROW_DEF_O, new Object[] { oid, cid,
						"Order_" + oid });
				assertEquals(OK, store.writeRow(rowO));
				for (int i = 0; ++i <= 10;) {
					int iid = oid * 1000 + i;
					rowI.reset(0, 64);
					rowI.createRow(ROW_DEF_I, new Object[] { iid, oid,
							"Item_" + iid });
					assertEquals(OK, store.writeRow(rowI));
				}
			}
		}

	}
	public void testSimpleScanCOIrows() throws Exception {
		rowDefCache.putRowDef(ROW_DEF_C);
		rowDefCache.putRowDef(ROW_DEF_O);
		rowDefCache.putRowDef(ROW_DEF_I);
		final RowData rowC = new RowData(new byte[64]);
		final RowData rowO = new RowData(new byte[64]);
		final RowData rowI = new RowData(new byte[64]);
		int insertCount = 0;
		int scanCount = 0;
		
		for (int c = 0; ++c <= 10;) {
			int cid = c;
			rowC.reset(0, 64);
			rowC.createRow(ROW_DEF_C, new Object[] { cid, "Customer_" + cid });
			assertEquals(OK, store.writeRow(rowC));
			insertCount++;
			for (int o = 0; ++o <= 10;) {
				int oid = cid * 1000 + o;
				rowO.reset(0, 64);
				rowO.createRow(ROW_DEF_O, new Object[] { oid, cid,
						"Order_" + oid });
				assertEquals(OK, store.writeRow(rowO));
				insertCount++;
				for (int i = 0; ++i <= 10;) {
					int iid = oid * 1000 + i;
					rowI.reset(0, 64);
					rowI.createRow(ROW_DEF_I, new Object[] { iid, oid,
							"Item_" + iid });
					assertEquals(OK, store.writeRow(rowI));
					insertCount++;
				}
			}
		}
		
		rowI.createRow(ROW_DEF_I, new Object[] {null, null, null});
		final byte[] columnBitMap = new byte[]{(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF};
		final RowCollector rc = store.newRowCollector(1111, rowI, rowI, columnBitMap);
		final ByteBuffer payload = ByteBuffer.allocate(256);

		while (rc.hasMore()) {
			payload.clear();
			while(rc.collectNextRow(payload));
			payload.flip();
			RowData rowData = new RowData(payload.array(), payload.position(), payload.limit());
			for (int p = rowData.getBufferStart(); p < rowData.getBufferEnd();) {
				rowData.prepareRow(p);
				p = rowData.getRowEnd();
				scanCount++;
			}
		}
		assertEquals(insertCount, scanCount);
	}
<commit_msg>We're now little-endian everywhere. Except for message headers. This week.<commit_after>package com.akiba.cserver.store;

import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import com.akiba.util.ByteBufferFactory;
import junit.framework.TestCase;

import com.akiba.cserver.CServerConstants;
import com.akiba.cserver.FieldDef;
import com.akiba.cserver.FieldType;
import com.akiba.cserver.RowData;
import com.akiba.cserver.RowDef;
import com.akiba.cserver.RowDefCache;
import com.akiba.cserver.CServerUtil;
import com.persistit.Key;
import com.persistit.Persistit;

public class PersistitStoreTest extends TestCase implements CServerConstants {

	private final static File DATA_PATH = new File("/tmp/data");

	private final static RowDef ROW_DEF0 = RowDef.createRowDef(1234,
			new FieldDef[] { new FieldDef("t1", FieldType.INT),
					new FieldDef("t2", FieldType.INT),
					new FieldDef("t3", FieldType.INT) }, "test",
			"test_group_table", new int[] { 0 });

	//
	// Highly abbreviate "COI" structure.
	//
	private final static RowDef ROW_DEF_C = RowDef.createRowDef(100,
			new FieldDef[] { new FieldDef("customer_id", FieldType.INT),
					new FieldDef("customer_x", FieldType.VARCHAR, 100) },
			"customers", "coi", new int[] { 0 });

	private final static RowDef ROW_DEF_O = RowDef.createRowDef(101,
			new FieldDef[] { new FieldDef("order_id", FieldType.INT),
					new FieldDef("date", FieldType.INT),
					new FieldDef("description", FieldType.VARCHAR, 100) },
			"orders", "coi", new int[] { 0 }, 100, new int[] { 1 });

	private final static RowDef ROW_DEF_I = RowDef.createRowDef(102,
			new FieldDef[] { new FieldDef("item_id", FieldType.INT),
					new FieldDef("qty", FieldType.INT),
					new FieldDef("description", FieldType.VARCHAR, 100) },
			"items", "coi", new int[] { 0 }, 101, new int[] { 1 });

	private PersistitStore store;

	private RowDefCache rowDefCache;

	@Override
	public void setUp() throws Exception {
		rowDefCache = new RowDefCache();
		store = new PersistitStore(rowDefCache);
		CServerUtil.cleanUpDirectory(DATA_PATH);
		PersistitStore.setDataPath(DATA_PATH.getPath());
		store.startUp();
	}

	@Override
	public void tearDown() throws Exception {
		store.shutDown();
		store = null;
		rowDefCache = null;
	}

	public void testWriteSimpleRootRow() throws Exception {
		rowDefCache.putRowDef(ROW_DEF0);
		final RowData rowData = new RowData(new byte[64]);
		rowData.createRow(ROW_DEF0, new Object[] { 1, 2, 3 });
		assertEquals(OK, store.writeRow(rowData));
	}

	public void testWriteCOIrows() throws Exception {
		rowDefCache.putRowDef(ROW_DEF_C);
		rowDefCache.putRowDef(ROW_DEF_O);
		rowDefCache.putRowDef(ROW_DEF_I);
		final RowData rowC = new RowData(new byte[64]);
		final RowData rowO = new RowData(new byte[64]);
		final RowData rowI = new RowData(new byte[64]);

		for (int c = 0; ++c <= 10;) {
			int cid = c;
			rowC.reset(0, 64);
			rowC.createRow(ROW_DEF_C, new Object[] { cid, "Customer_" + cid });
			assertEquals(OK, store.writeRow(rowC));
			for (int o = 0; ++o <= 10;) {
				int oid = cid * 1000 + o;
				rowO.reset(0, 64);
				rowO.createRow(ROW_DEF_O, new Object[] { oid, cid,
						"Order_" + oid });
				assertEquals(OK, store.writeRow(rowO));
				for (int i = 0; ++i <= 10;) {
					int iid = oid * 1000 + i;
					rowI.reset(0, 64);
					rowI.createRow(ROW_DEF_I, new Object[] { iid, oid,
							"Item_" + iid });
					assertEquals(OK, store.writeRow(rowI));
				}
			}
		}

	}
	public void testSimpleScanCOIrows() throws Exception {
		rowDefCache.putRowDef(ROW_DEF_C);
		rowDefCache.putRowDef(ROW_DEF_O);
		rowDefCache.putRowDef(ROW_DEF_I);
		final RowData rowC = new RowData(new byte[64]);
		final RowData rowO = new RowData(new byte[64]);
		final RowData rowI = new RowData(new byte[64]);
		int insertCount = 0;
		int scanCount = 0;
		
		for (int c = 0; ++c <= 10;) {
			int cid = c;
			rowC.reset(0, 64);
			rowC.createRow(ROW_DEF_C, new Object[] { cid, "Customer_" + cid });
			assertEquals(OK, store.writeRow(rowC));
			insertCount++;
			for (int o = 0; ++o <= 10;) {
				int oid = cid * 1000 + o;
				rowO.reset(0, 64);
				rowO.createRow(ROW_DEF_O, new Object[] { oid, cid,
						"Order_" + oid });
				assertEquals(OK, store.writeRow(rowO));
				insertCount++;
				for (int i = 0; ++i <= 10;) {
					int iid = oid * 1000 + i;
					rowI.reset(0, 64);
					rowI.createRow(ROW_DEF_I, new Object[] { iid, oid,
							"Item_" + iid });
					assertEquals(OK, store.writeRow(rowI));
					insertCount++;
				}
			}
		}
		
		rowI.createRow(ROW_DEF_I, new Object[] {null, null, null});
		final byte[] columnBitMap = new byte[]{(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF};
		final RowCollector rc = store.newRowCollector(1111, rowI, rowI, columnBitMap);
		final ByteBuffer payload = ByteBufferFactory.allocate(256);

		while (rc.hasMore()) {
			payload.clear();
			while(rc.collectNextRow(payload));
			payload.flip();
			RowData rowData = new RowData(payload.array(), payload.position(), payload.limit());
			for (int p = rowData.getBufferStart(); p < rowData.getBufferEnd();) {
				rowData.prepareRow(p);
				p = rowData.getRowEnd();
				scanCount++;
			}
		}
		assertEquals(insertCount, scanCount);
	}

{'fraction_non_alphanumeric': 0.14658614980033116, 'fraction_numerical': 0.02376546216031947, 'mean_word_length': 3.9532079112397494, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6744107', 'n_tokens_mistral': 4361, 'n_tokens_neox': 3661, 'n_words': 869}