<commit_before>/*
 * Copyright 2013 bits of proof zrt.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bitsofproof.supernode.core;

import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.bitsofproof.supernode.common.BloomFilter;
import com.bitsofproof.supernode.common.BloomFilter.UpdateMode;
import com.bitsofproof.supernode.common.ByteUtils;
import com.bitsofproof.supernode.common.ByteVector;
import com.bitsofproof.supernode.common.ExtendedKey;
import com.bitsofproof.supernode.common.Hash;
import com.bitsofproof.supernode.common.ScriptFormat;
import com.bitsofproof.supernode.common.ScriptFormat.Token;
import com.bitsofproof.supernode.common.ValidationException;
import com.bitsofproof.supernode.common.WireFormat;
import com.bitsofproof.supernode.model.Blk;
import com.bitsofproof.supernode.model.Head;
import com.bitsofproof.supernode.model.Tx;
import com.bitsofproof.supernode.model.TxIn;
import com.bitsofproof.supernode.model.TxOut;

public abstract class CachedBlockStore implements BlockStore
{
	private static final Logger log = LoggerFactory.getLogger (CachedBlockStore.class);

	private static final long MAX_BLOCK_SIGOPS = 20000;

	// not allowed to branch further back on trunk
	private static final int FORCE_TRUNK = 100;
	private static final long MIN_RELAY_TX_FEE = 10000;
	private static final long KB_RELAY_TX_FEE = 50000;
	private static final int COINBASE_MATURITY = 100;

	private static final Map<Integer, String> checkPoints = new HashMap<Integer, String> ();
	private static final int lastCheckPoint;

	static
	{
		checkPoints.put (11111, "0000000069e244f73d78e8fd29ba2fd2ed618bd6fa2ee92559f542fdb26e7c1d");
		checkPoints.put (33333, "000000002dd5588a74784eaa7ab0507a18ad16a236e7b1ce69f00d7ddfb5d0a6");
		checkPoints.put (74000, "0000000000573993a3c9e41ce34471c079dcf5f52a0e824a81e7f953b8661a20");
		checkPoints.put (105000, "00000000000291ce28027faea320c8d2b054b2e0fe44a773f3eefb151d6bdc97");
		checkPoints.put (134444, "00000000000005b12ffd4cd315cd34ffd4a594f430ac814c91184a0d42d2b0fe");
		checkPoints.put (168000, "000000000000099e61ea72015e79632f216fe6cb33d7899acb35b75c8303b763");
		checkPoints.put (193000, "000000000000059f452a5f7340de6682a977387c17010ff6e6c3bd83ca8b1317");
		checkPoints.put (210000, "000000000000048b95347e83192f69cf0366076336c639f9b7228e9ba171342e");
		checkPoints.put (222222, "00000000000000b8b49d0b61b14994b5c0a511c4b48a1e251ff2b479b2e6f678");
		checkPoints.put (232000, "000000000000018f47636e1c3a946db77624880ae484ffb0233f5aac6316b3bb");
		checkPoints.put (270000, "0000000000000002a775aec59dc6a9e4bb1c025cf1b8c2195dd9dc3998c827c5");

		lastCheckPoint = 270000;
	}

	protected Chain chain;

	private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock ();

	private boolean enforceV2Block = false;

	protected CachedHead currentHead = null;
	protected final Map<String, CachedBlock> cachedBlocks = new HashMap<String, CachedBlock> ();
	protected final Map<Long, CachedHead> cachedHeads = new HashMap<Long, CachedHead> ();

	private final LinkedList<CachedBlock> blockChain = new LinkedList<CachedBlock> ();

	private final ImplementTxOutCache cachedUTXO = new ImplementTxOutCache ();
	private final List<TrunkListener> trunkListener = new ArrayList<TrunkListener> ();

	private final ExecutorService inputProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);
	private final ExecutorService transactionsProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);

	@Override
	public ValidationException runInCacheContext (CacheContextRunnable runnable)
	{
		try
		{
			lock.readLock ().lock ();

			runnable.run (cachedUTXO);
			return null;
		}
		catch ( ValidationException e )
		{
			return e;
		}
		finally
		{
			lock.readLock ().unlock ();
		}
	}

	protected abstract void clearStore ();

	protected abstract void startBatch ();

	protected abstract void endBatch ();

	protected abstract void cancelBatch ();

	protected abstract void cacheChain ();

	protected abstract void cacheHeads ();

	protected abstract void cacheColors ();

	protected abstract void cacheUTXO (int lookback, TxOutCache cache);

	protected abstract List<TxOut> findTxOuts (Map<String, HashSet<Long>> need) throws ValidationException;

	protected abstract void checkBIP30Compliance (Set<String> txs, int untilHeight) throws ValidationException;

	protected abstract void backwardCache (Blk b, TxOutCache cache, boolean modify) throws ValidationException;

	protected abstract void forwardCache (Blk b, TxOutCache cache, boolean modify) throws ValidationException;

	protected abstract TxOut getSourceReference (TxOut source) throws ValidationException;

	protected abstract void insertBlock (Blk b) throws ValidationException;

	protected abstract void insertHead (Head head);

	protected abstract Head updateHead (Head head);

	protected abstract Head retrieveHead (CachedHead cached) throws ValidationException;

	protected abstract Blk retrieveBlock (CachedBlock cached) throws ValidationException;

	protected abstract Blk retrieveBlockHeader (CachedBlock cached) throws ValidationException;

	protected abstract void updateColor (TxOut root, String fungibleName) throws ValidationException;

	@Override
	public void addTrunkListener (TrunkListener listener)
	{
		trunkListener.add (listener);
	}

	protected static class CachedHead
	{
		private Long id;
		private CachedBlock last;
		private long chainWork;
		private int height;
		private CachedHead previous;
		private int previousHeight;
		private final Set<CachedBlock> blocks = new HashSet<CachedBlock> ();

		public CachedHead ()
		{
		}

		public Long getId ()
		{
			return id;
		}

		public void setId (Long id)
		{
			this.id = id;
		}

		public long getChainWork ()
		{
			return chainWork;
		}

		public long getHeight ()
		{
			return height;
		}

		public void setChainWork (long chainWork)
		{
			this.chainWork = chainWork;
		}

		public void setHeight (int height)
		{
			this.height = height;
		}

		public Set<CachedBlock> getBlocks ()
		{
			return blocks;
		}

		public CachedHead getPrevious ()
		{
			return previous;
		}

		public void setPrevious (CachedHead previous)
		{
			this.previous = previous;
		}

		public CachedBlock getLast ()
		{
			return last;
		}

		public void setLast (CachedBlock last)
		{
			this.last = last;
		}

		public int getPreviousHeight ()
		{
			return previousHeight;
		}

		public void setPreviousHeight (int previousHeight)
		{
			this.previousHeight = previousHeight;
		}

	}

	protected static class CachedBlock
	{
		public CachedBlock (String hash, Long id, CachedBlock previous, long time, int height, int version, byte[] filterMap, int filterFunctions)
		{
			this.hash = hash;
			this.id = id;
			this.previous = previous;
			this.time = time;
			this.height = height;
			this.version = version;
			this.filterMap = filterMap;
			this.filterFunctions = filterFunctions;
		}

		private final String hash;
		private final Long id;
		private final CachedBlock previous;
		private final long time;
		private final int height;
		private final int version;
		private final byte[] filterMap;
		private final int filterFunctions;

		public byte[] getFilterMap ()
		{
			return filterMap;
		}

		public int getFilterFunctions ()
		{
			return filterFunctions;
		}

		public Long getId ()
		{
			return id;
		}

		public CachedBlock getPrevious ()
		{
			return previous;
		}

		public long getTime ()
		{
			return time;
		}

		public String getHash ()
		{
			return hash;
		}

		public int getHeight ()
		{
			return height;
		}

		public int getVersion ()
		{
			return version;
		}

		@Override
		public int hashCode ()
		{
			return hash.hashCode ();
		}

		@Override
		public boolean equals (Object o)
		{
			if ( o == null )
			{
				return false;
			}
			if ( o == this )
			{
				return true;
			}
			if ( o instanceof CachedBlock )
			{
				return hash.equals (((CachedBlock) o).hash);
			}
			return false;
		}
	}

	@Override
	public void cache (Chain chain, int size) throws ValidationException
	{
		this.chain = chain;

		try
		{
			lock.writeLock ().lock ();
			cachedBlocks.clear ();
			cachedHeads.clear ();
			blockChain.clear ();
			cachedUTXO.clear ();

			log.trace ("Cache heads...");
			cacheHeads ();

			log.trace ("Cache colors...");
			cacheColors ();

			log.trace ("Cache chain...");
			cacheChain ();

			updateBlockChainCache ();

			if ( size > 0 )
			{
				log.trace ("Cache UTXO set ...");
				cacheUTXO (size, cachedUTXO);
			}

			log.trace ("Cache filled.");
		}
		finally
		{
			lock.writeLock ().unlock ();
		}
	}

	@Override
	public void catchUp (String hash, List<Blk> added, List<Blk> removed) throws ValidationException
	{
		try
		{
			lock.readLock ().lock ();

			String trunk = hash;
			while ( !isOnTrunk (trunk) )
			{
				CachedBlock b = cachedBlocks.get (trunk);
				removed.add (retrieveBlock (b));
				b = b.previous;
				trunk = b.hash;
			}
			CachedBlock q = currentHead.getLast ();
			CachedBlock p = q.previous;
			while ( p != null )
			{
				if ( q.hash.equals (trunk) )
				{
					break;
				}
				added.add (retrieveBlock (q));
				p = q.previous;
			}
			Collections.reverse (added);
		}
		finally
		{
			lock.readLock ().unlock ();
		}

	}
<commit_msg>log catchup processing<commit_after>/*
 * Copyright 2013 bits of proof zrt.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bitsofproof.supernode.core;

import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.bitsofproof.supernode.common.BloomFilter;
import com.bitsofproof.supernode.common.BloomFilter.UpdateMode;
import com.bitsofproof.supernode.common.ByteUtils;
import com.bitsofproof.supernode.common.ByteVector;
import com.bitsofproof.supernode.common.ExtendedKey;
import com.bitsofproof.supernode.common.Hash;
import com.bitsofproof.supernode.common.ScriptFormat;
import com.bitsofproof.supernode.common.ScriptFormat.Token;
import com.bitsofproof.supernode.common.ValidationException;
import com.bitsofproof.supernode.common.WireFormat;
import com.bitsofproof.supernode.model.Blk;
import com.bitsofproof.supernode.model.Head;
import com.bitsofproof.supernode.model.Tx;
import com.bitsofproof.supernode.model.TxIn;
import com.bitsofproof.supernode.model.TxOut;

public abstract class CachedBlockStore implements BlockStore
{
	private static final Logger log = LoggerFactory.getLogger (CachedBlockStore.class);

	private static final long MAX_BLOCK_SIGOPS = 20000;

	// not allowed to branch further back on trunk
	private static final int FORCE_TRUNK = 100;
	private static final long MIN_RELAY_TX_FEE = 10000;
	private static final long KB_RELAY_TX_FEE = 50000;
	private static final int COINBASE_MATURITY = 100;

	private static final Map<Integer, String> checkPoints = new HashMap<Integer, String> ();
	private static final int lastCheckPoint;

	static
	{
		checkPoints.put (11111, "0000000069e244f73d78e8fd29ba2fd2ed618bd6fa2ee92559f542fdb26e7c1d");
		checkPoints.put (33333, "000000002dd5588a74784eaa7ab0507a18ad16a236e7b1ce69f00d7ddfb5d0a6");
		checkPoints.put (74000, "0000000000573993a3c9e41ce34471c079dcf5f52a0e824a81e7f953b8661a20");
		checkPoints.put (105000, "00000000000291ce28027faea320c8d2b054b2e0fe44a773f3eefb151d6bdc97");
		checkPoints.put (134444, "00000000000005b12ffd4cd315cd34ffd4a594f430ac814c91184a0d42d2b0fe");
		checkPoints.put (168000, "000000000000099e61ea72015e79632f216fe6cb33d7899acb35b75c8303b763");
		checkPoints.put (193000, "000000000000059f452a5f7340de6682a977387c17010ff6e6c3bd83ca8b1317");
		checkPoints.put (210000, "000000000000048b95347e83192f69cf0366076336c639f9b7228e9ba171342e");
		checkPoints.put (222222, "00000000000000b8b49d0b61b14994b5c0a511c4b48a1e251ff2b479b2e6f678");
		checkPoints.put (232000, "000000000000018f47636e1c3a946db77624880ae484ffb0233f5aac6316b3bb");
		checkPoints.put (270000, "0000000000000002a775aec59dc6a9e4bb1c025cf1b8c2195dd9dc3998c827c5");

		lastCheckPoint = 270000;
	}

	protected Chain chain;

	private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock ();

	private boolean enforceV2Block = false;

	protected CachedHead currentHead = null;
	protected final Map<String, CachedBlock> cachedBlocks = new HashMap<String, CachedBlock> ();
	protected final Map<Long, CachedHead> cachedHeads = new HashMap<Long, CachedHead> ();

	private final LinkedList<CachedBlock> blockChain = new LinkedList<CachedBlock> ();

	private final ImplementTxOutCache cachedUTXO = new ImplementTxOutCache ();
	private final List<TrunkListener> trunkListener = new ArrayList<TrunkListener> ();

	private final ExecutorService inputProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);
	private final ExecutorService transactionsProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);

	@Override
	public ValidationException runInCacheContext (CacheContextRunnable runnable)
	{
		try
		{
			lock.readLock ().lock ();

			runnable.run (cachedUTXO);
			return null;
		}
		catch ( ValidationException e )
		{
			return e;
		}
		finally
		{
			lock.readLock ().unlock ();
		}
	}

	protected abstract void clearStore ();

	protected abstract void startBatch ();

	protected abstract void endBatch ();

	protected abstract void cancelBatch ();

	protected abstract void cacheChain ();

	protected abstract void cacheHeads ();

	protected abstract void cacheColors ();

	protected abstract void cacheUTXO (int lookback, TxOutCache cache);

	protected abstract List<TxOut> findTxOuts (Map<String, HashSet<Long>> need) throws ValidationException;

	protected abstract void checkBIP30Compliance (Set<String> txs, int untilHeight) throws ValidationException;

	protected abstract void backwardCache (Blk b, TxOutCache cache, boolean modify) throws ValidationException;

	protected abstract void forwardCache (Blk b, TxOutCache cache, boolean modify) throws ValidationException;

	protected abstract TxOut getSourceReference (TxOut source) throws ValidationException;

	protected abstract void insertBlock (Blk b) throws ValidationException;

	protected abstract void insertHead (Head head);

	protected abstract Head updateHead (Head head);

	protected abstract Head retrieveHead (CachedHead cached) throws ValidationException;

	protected abstract Blk retrieveBlock (CachedBlock cached) throws ValidationException;

	protected abstract Blk retrieveBlockHeader (CachedBlock cached) throws ValidationException;

	protected abstract void updateColor (TxOut root, String fungibleName) throws ValidationException;

	@Override
	public void addTrunkListener (TrunkListener listener)
	{
		trunkListener.add (listener);
	}

	protected static class CachedHead
	{
		private Long id;
		private CachedBlock last;
		private long chainWork;
		private int height;
		private CachedHead previous;
		private int previousHeight;
		private final Set<CachedBlock> blocks = new HashSet<CachedBlock> ();

		public CachedHead ()
		{
		}

		public Long getId ()
		{
			return id;
		}

		public void setId (Long id)
		{
			this.id = id;
		}

		public long getChainWork ()
		{
			return chainWork;
		}

		public long getHeight ()
		{
			return height;
		}

		public void setChainWork (long chainWork)
		{
			this.chainWork = chainWork;
		}

		public void setHeight (int height)
		{
			this.height = height;
		}

		public Set<CachedBlock> getBlocks ()
		{
			return blocks;
		}

		public CachedHead getPrevious ()
		{
			return previous;
		}

		public void setPrevious (CachedHead previous)
		{
			this.previous = previous;
		}

		public CachedBlock getLast ()
		{
			return last;
		}

		public void setLast (CachedBlock last)
		{
			this.last = last;
		}

		public int getPreviousHeight ()
		{
			return previousHeight;
		}

		public void setPreviousHeight (int previousHeight)
		{
			this.previousHeight = previousHeight;
		}

	}

	protected static class CachedBlock
	{
		public CachedBlock (String hash, Long id, CachedBlock previous, long time, int height, int version, byte[] filterMap, int filterFunctions)
		{
			this.hash = hash;
			this.id = id;
			this.previous = previous;
			this.time = time;
			this.height = height;
			this.version = version;
			this.filterMap = filterMap;
			this.filterFunctions = filterFunctions;
		}

		private final String hash;
		private final Long id;
		private final CachedBlock previous;
		private final long time;
		private final int height;
		private final int version;
		private final byte[] filterMap;
		private final int filterFunctions;

		public byte[] getFilterMap ()
		{
			return filterMap;
		}

		public int getFilterFunctions ()
		{
			return filterFunctions;
		}

		public Long getId ()
		{
			return id;
		}

		public CachedBlock getPrevious ()
		{
			return previous;
		}

		public long getTime ()
		{
			return time;
		}

		public String getHash ()
		{
			return hash;
		}

		public int getHeight ()
		{
			return height;
		}

		public int getVersion ()
		{
			return version;
		}

		@Override
		public int hashCode ()
		{
			return hash.hashCode ();
		}

		@Override
		public boolean equals (Object o)
		{
			if ( o == null )
			{
				return false;
			}
			if ( o == this )
			{
				return true;
			}
			if ( o instanceof CachedBlock )
			{
				return hash.equals (((CachedBlock) o).hash);
			}
			return false;
		}
	}

	@Override
	public void cache (Chain chain, int size) throws ValidationException
	{
		this.chain = chain;

		try
		{
			lock.writeLock ().lock ();
			cachedBlocks.clear ();
			cachedHeads.clear ();
			blockChain.clear ();
			cachedUTXO.clear ();

			log.trace ("Cache heads...");
			cacheHeads ();

			log.trace ("Cache colors...");
			cacheColors ();

			log.trace ("Cache chain...");
			cacheChain ();

			updateBlockChainCache ();

			if ( size > 0 )
			{
				log.trace ("Cache UTXO set ...");
				cacheUTXO (size, cachedUTXO);
			}

			log.trace ("Cache filled.");
		}
		finally
		{
			lock.writeLock ().unlock ();
		}
	}

	@Override
	public void catchUp (String hash, List<Blk> added, List<Blk> removed) throws ValidationException
	{
		try
		{
			lock.readLock ().lock ();

			String trunk = hash;
			while ( !isOnTrunk (trunk) )
			{
				log.debug ("need to remove: " + trunk);
				CachedBlock b = cachedBlocks.get (trunk);
				removed.add (retrieveBlock (b));
				b = b.previous;
				trunk = b.hash;
			}
			CachedBlock q = currentHead.getLast ();
			CachedBlock p = q.previous;
			while ( p != null )
			{
				if ( q.hash.equals (trunk) )
				{
					break;
				}
				log.debug ("need to add: " + trunk);
				added.add (retrieveBlock (q));
				p = q.previous;
			}
			Collections.reverse (added);
		}
		finally
		{
			lock.readLock ().unlock ();
		}

	}

{'fraction_non_alphanumeric': 0.08941358622024385, 'fraction_numerical': 0.057867234371976, 'mean_word_length': 4.21812673567281, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 22, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4629050', 'n_tokens_mistral': 8066, 'n_tokens_neox': 6515, 'n_words': 1694}