<commit_before>package org.exist.storage.store;

import java.io.IOException;
import java.util.Iterator;

import org.apache.log4j.Logger;
import org.dbxml.core.filer.BTree;
import org.dbxml.core.filer.BTreeException;
import org.exist.dom.DocumentImpl;
import org.exist.dom.NodeImpl;
import org.exist.dom.NodeProxy;
import org.exist.util.ByteConversion;
import org.exist.util.Lock;
import org.exist.util.LockException;

/**
 * Class NodeIterator is used to iterate over nodes in the DOM storage.
 * This implementation locks the DOM file to read the node and unlocks
 * it afterwards. It is thus safer than DOMFileIterator, since the node's
 * value will not change. 
 * 
 * @author wolf
 */
public final class NodeIterator implements Iterator {

	private final static Logger LOG = Logger.getLogger(NodeIterator.class);

	private DOMFile db = null;
	private NodeProxy node = null;
	private DocumentImpl doc = null;
	private int offset;
	private short lastTID = -1;
	private DOMFile.DOMPage p = null;
	private long page;
	private long startAddress = -1;
	private Object lockKey;
	private boolean useNodePool = false;

	public NodeIterator(Object lock, DOMFile db, NodeProxy node, boolean poolable)
		throws BTreeException, IOException {
		this.db = db;
		this.doc = node.doc;
		this.useNodePool = poolable;
		if (-1 < node.getInternalAddress())
			startAddress = node.getInternalAddress();
		else
			this.node = node;
		lockKey = (lock == null ? this : lock);
	}

	public NodeIterator(Object lock, DOMFile db, DocumentImpl doc, long address) {
		this.db = db;
		this.doc = doc;
		this.startAddress = address;
		lockKey = (lock == null ? this : lock);
	}
	
	/**
	 *  Returns the internal virtual address of the node at the iterator's
	 * current position.
	 *
	 *@return    The currentAddress value
	 */
	public long currentAddress() {
		return StorageAddress.createPointer((int) page, ItemId.getId(lastTID));
	}

	/**
	 *  Are there more nodes to be read?
	 *
	 *@return    Description of the Return Value
	 */
	public boolean hasNext() {
		Lock lock = db.getLock();
		try {
			try {
				lock.acquire();
			} catch (LockException e) {
				return false;
			}
			if(gotoNextPosition()) {
				db.getPageBuffer().add(p);
				final DOMFile.DOMFilePageHeader ph = p.getPageHeader();
				if (offset < ph.getDataLength())
					return true;
				else if (ph.getNextDataPage() < 0)
					return false;
				else
					return true;
			}
		} catch (BTreeException e) {
			LOG.warn(e);
		} catch (IOException e) {
			LOG.warn(e);
		} finally {
			lock.release();
		}
		return false;
	}

	/**
	 *  Returns the next node in document order. 
	 */
	public Object next() {
		Lock lock = db.getLock();
		NodeImpl nextNode = null;
		try {
			try {
				lock.acquire(Lock.READ_LOCK);
			} catch (LockException e) {
				return null;
			}
			if(gotoNextPosition()) {
			    boolean skipped = false;
			    do {
					DOMFile.DOMFilePageHeader ph = p.getPageHeader();
					// next value larger than length of the current page?
					if (offset >= ph.getDataLength()) {
						// load next page in chain
						long nextPage = ph.getNextDataPage();
						if (nextPage < 0) {
							LOG.debug("bad link to next " + p.page.getPageInfo() + "; previous: " +
									ph.getPrevDataPage());
							return null;
						}
						page = nextPage;
						p = db.getCurrentPage(nextPage);
						db.addToBuffer(p);
						offset = 0;
					}
					// extract the tid
					lastTID = ByteConversion.byteToShort(p.data, offset);
					offset += 2;
					//	check if this is just a link to a relocated node
					if(ItemId.isLink(lastTID)) {
						// skip this
						long link = ByteConversion.byteToLong(p.data, offset);
						offset += 8;
//						System.out.println("skipping link on p " + page + " -> " + 
//								StorageAddress.pageFromPointer(link));
						skipped = true;
						continue;
					}
					// read data length
					short l = ByteConversion.byteToShort(p.data, offset);
					offset += 2;
					if(ItemId.isRelocated(lastTID)) {
						// found a relocated node. Skip the next 8 bytes
						offset += 8;
					}
					//	overflow page? load the overflow value
					if(l == DOMFile.OVERFLOW) {
					    LOG.warn("unexpected overflow page at " + p.getPageNum() + "; tid = " + 
					            ItemId.getId(lastTID) + "; offset = " + offset + "; skipped = " + skipped);
						l = 8;
						final long overflow = ByteConversion.byteToLong(p.data, offset);
						offset += 8;
						final byte[] odata = db.getOverflowValue(overflow);<commit_msg>Removed misleading log message: "unexpected overflow page ...".<commit_after>package org.exist.storage.store;

import java.io.IOException;
import java.util.Iterator;

import org.apache.log4j.Logger;
import org.dbxml.core.filer.BTree;
import org.dbxml.core.filer.BTreeException;
import org.exist.dom.DocumentImpl;
import org.exist.dom.NodeImpl;
import org.exist.dom.NodeProxy;
import org.exist.util.ByteConversion;
import org.exist.util.Lock;
import org.exist.util.LockException;

/**
 * Class NodeIterator is used to iterate over nodes in the DOM storage.
 * This implementation locks the DOM file to read the node and unlocks
 * it afterwards. It is thus safer than DOMFileIterator, since the node's
 * value will not change. 
 * 
 * @author wolf
 */
public final class NodeIterator implements Iterator {

	private final static Logger LOG = Logger.getLogger(NodeIterator.class);

	private DOMFile db = null;
	private NodeProxy node = null;
	private DocumentImpl doc = null;
	private int offset;
	private short lastTID = -1;
	private DOMFile.DOMPage p = null;
	private long page;
	private long startAddress = -1;
	private Object lockKey;
	private boolean useNodePool = false;

	public NodeIterator(Object lock, DOMFile db, NodeProxy node, boolean poolable)
		throws BTreeException, IOException {
		this.db = db;
		this.doc = node.doc;
		this.useNodePool = poolable;
		if (-1 < node.getInternalAddress())
			startAddress = node.getInternalAddress();
		else
			this.node = node;
		lockKey = (lock == null ? this : lock);
	}

	public NodeIterator(Object lock, DOMFile db, DocumentImpl doc, long address) {
		this.db = db;
		this.doc = doc;
		this.startAddress = address;
		lockKey = (lock == null ? this : lock);
	}
	
	/**
	 *  Returns the internal virtual address of the node at the iterator's
	 * current position.
	 *
	 *@return    The currentAddress value
	 */
	public long currentAddress() {
		return StorageAddress.createPointer((int) page, ItemId.getId(lastTID));
	}

	/**
	 *  Are there more nodes to be read?
	 *
	 *@return    Description of the Return Value
	 */
	public boolean hasNext() {
		Lock lock = db.getLock();
		try {
			try {
				lock.acquire();
			} catch (LockException e) {
				return false;
			}
			if(gotoNextPosition()) {
				db.getPageBuffer().add(p);
				final DOMFile.DOMFilePageHeader ph = p.getPageHeader();
				if (offset < ph.getDataLength())
					return true;
				else if (ph.getNextDataPage() < 0)
					return false;
				else
					return true;
			}
		} catch (BTreeException e) {
			LOG.warn(e);
		} catch (IOException e) {
			LOG.warn(e);
		} finally {
			lock.release();
		}
		return false;
	}

	/**
	 *  Returns the next node in document order. 
	 */
	public Object next() {
		Lock lock = db.getLock();
		NodeImpl nextNode = null;
		try {
			try {
				lock.acquire(Lock.READ_LOCK);
			} catch (LockException e) {
				return null;
			}
			if(gotoNextPosition()) {
			    boolean skipped = false;
			    do {
					DOMFile.DOMFilePageHeader ph = p.getPageHeader();
					// next value larger than length of the current page?
					if (offset >= ph.getDataLength()) {
						// load next page in chain
						long nextPage = ph.getNextDataPage();
						if (nextPage < 0) {
							LOG.debug("bad link to next " + p.page.getPageInfo() + "; previous: " +
									ph.getPrevDataPage());
							return null;
						}
						page = nextPage;
						p = db.getCurrentPage(nextPage);
						db.addToBuffer(p);
						offset = 0;
					}
					// extract the tid
					lastTID = ByteConversion.byteToShort(p.data, offset);
					offset += 2;
					//	check if this is just a link to a relocated node
					if(ItemId.isLink(lastTID)) {
						// skip this
						long link = ByteConversion.byteToLong(p.data, offset);
						offset += 8;
//						System.out.println("skipping link on p " + page + " -> " + 
//								StorageAddress.pageFromPointer(link));
						skipped = true;
						continue;
					}
					// read data length
					short l = ByteConversion.byteToShort(p.data, offset);
					offset += 2;
					if(ItemId.isRelocated(lastTID)) {
						// found a relocated node. Skip the next 8 bytes
						offset += 8;
					}
					//	overflow page? load the overflow value
					if(l == DOMFile.OVERFLOW) {
//					    LOG.warn("unexpected overflow page at " + p.getPageNum() + "; tid = " + 
//					            ItemId.getId(lastTID) + "; offset = " + offset + "; skipped = " + skipped);
						l = 8;
						final long overflow = ByteConversion.byteToLong(p.data, offset);
						offset += 8;
						final byte[] odata = db.getOverflowValue(overflow);
{'fraction_non_alphanumeric': 0.11586985391766268, 'fraction_numerical': 0.003098716246126605, 'mean_word_length': 2.8066554338668914, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5098925', 'n_tokens_mistral': 3470, 'n_tokens_neox': 2799, 'n_words': 824}