<commit_before>/*
 * Copyright 2012 <NAME> <EMAIL>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bitsofproof.supernode.model;

import java.math.BigInteger;
import java.security.acl.Owner;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import com.bitsofproof.supernode.core.AddressConverter;
import com.bitsofproof.supernode.core.Chain;
import com.bitsofproof.supernode.core.Difficulty;
import com.bitsofproof.supernode.core.Hash;
import com.bitsofproof.supernode.core.Script;
import com.bitsofproof.supernode.core.Script.Opcode;
import com.bitsofproof.supernode.core.ValidationException;
import com.bitsofproof.supernode.core.WireFormat;
import com.mysema.query.jpa.impl.JPAQuery;

@Component ("jpaBlockStore")
class JpaBlockStore implements BlockStore
{
	private static final Logger log = LoggerFactory.getLogger (JpaBlockStore.class);

	private static final long MAX_BLOCK_SIGOPS = 20000;

	@Autowired
	private Chain chain;

	@PersistenceContext
	private EntityManager entityManager;

	@Autowired
	private PlatformTransactionManager transactionManager;

	// note: the lock protects the caches (currentHead, cachedBlocks, cachedHeads, utxoCache) not the database
	private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock ();

	private CachedHead currentHead = null;
	private final Map<String, CachedBlock> cachedBlocks = new HashMap<String, CachedBlock> ();
	private final Map<Long, CachedHead> cachedHeads = new HashMap<Long, CachedHead> ();

	private final Cache utxoCache = new Cache ();
	private final ExecutorService inputProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);
	private final ExecutorService transactionsProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);

	private class Cache
	{
		private final Map<String, Tx> txCache = new HashMap<String, Tx> ();

		private final Map<String, BigInteger> outputs = new HashMap<String, BigInteger> ();

		public void remove (String key, BigInteger c)
		{
			BigInteger mask = outputs.get (key);
			if ( mask != null )
			{
				mask = mask.xor (c);
				outputs.put (key, mask);

				if ( mask.equals (BigInteger.ZERO) )
				{
					txCache.remove (key);
				}
			}
		}

		public void add (String key, BigInteger c)
		{
			BigInteger mask = outputs.get (key);
			if ( mask == null )
			{
				mask = BigInteger.ZERO;
			}
			outputs.put (key, mask.or (c));
		}

		public List<TxOut> get (String txhash)
		{
			List<TxOut> outs = new ArrayList<TxOut> ();
			Tx cached = txCache.get (txhash);
			if ( cached != null )
			{
				for ( TxOut out : cached.getOutputs () )
				{
					if ( contains (txhash, out.getIx ()) )
					{
						outs.add (out);
					}
				}
			}
			return outs;
		}

		public boolean contains (String key, long ix)
		{
			BigInteger mask = outputs.get (key);
			if ( mask == null )
			{
				return false;
			}
			return mask.testBit ((int) ix);
		}

		public void add (Blk b)
		{
			for ( Tx t : b.getTransactions () )
			{
				if ( outputs.containsKey (t.getHash ()) )
				{
					txCache.put (t.getHash (), t.flatCopy ());
				}
				for ( TxIn in : t.getInputs () )
				{
					if ( !outputs.containsKey (in.getSourceHash ()) )
					{
						txCache.remove (in.getSourceHash ());
					}
				}
			}
		}

		public void remove (Blk b)
		{
			for ( Tx t : b.getTransactions () )
			{
				if ( !outputs.containsKey (t.getHash ()) )
				{
					txCache.remove (t.getHash ());
				}
				for ( TxIn in : t.getInputs () )
				{
					if ( outputs.containsKey (in.getSourceHash ()) )
					{
						txCache.put (in.getSourceHash (), in.getSource ().getTransaction ().flatCopy ());
					}
				}
			}
		}

		public void cacheUTXO ()
		{
			if ( !outputs.keySet ().isEmpty () )
			{
				QTx tx = QTx.tx;
				List<String> txs = new ArrayList<String> ();
				txs.addAll (outputs.keySet ());
				log.trace ("... total " + txs.size ());
				JPAQuery q = new JPAQuery (entityManager);
				int n = 0;
				for ( Tx t : q.from (tx).list (tx) )
				{
					if ( outputs.containsKey (t.getHash ()) )
					{
						txCache.put (t.getHash (), t.flatCopy ());
						entityManager.detach (t);
						if ( ++n % 10000 == 0 )
						{
							log.trace ("... read " + n);
						}
					}
				}
			}
		}
	}

	private static class CachedHead
	{
		private Long id;
		private CachedBlock last;
		private double chainWork;
		private long height;
		private CachedHead previous;
		private final Set<CachedBlock> blocks = new HashSet<CachedBlock> ();
<commit_msg>trying bacth read of cache<commit_after>/*
 * Copyright 2012 <NAME> <EMAIL>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bitsofproof.supernode.model;

import java.math.BigInteger;
import java.security.acl.Owner;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import com.bitsofproof.supernode.core.AddressConverter;
import com.bitsofproof.supernode.core.Chain;
import com.bitsofproof.supernode.core.Difficulty;
import com.bitsofproof.supernode.core.Hash;
import com.bitsofproof.supernode.core.Script;
import com.bitsofproof.supernode.core.Script.Opcode;
import com.bitsofproof.supernode.core.ValidationException;
import com.bitsofproof.supernode.core.WireFormat;
import com.mysema.query.jpa.impl.JPAQuery;

@Component ("jpaBlockStore")
class JpaBlockStore implements BlockStore
{
	private static final Logger log = LoggerFactory.getLogger (JpaBlockStore.class);

	private static final long MAX_BLOCK_SIGOPS = 20000;

	@Autowired
	private Chain chain;

	@PersistenceContext
	private EntityManager entityManager;

	@Autowired
	private PlatformTransactionManager transactionManager;

	// note: the lock protects the caches (currentHead, cachedBlocks, cachedHeads, utxoCache) not the database
	private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock ();

	private CachedHead currentHead = null;
	private final Map<String, CachedBlock> cachedBlocks = new HashMap<String, CachedBlock> ();
	private final Map<Long, CachedHead> cachedHeads = new HashMap<Long, CachedHead> ();

	private final Cache utxoCache = new Cache ();
	private final ExecutorService inputProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);
	private final ExecutorService transactionsProcessor = Executors.newFixedThreadPool (Runtime.getRuntime ().availableProcessors () * 2);

	private class Cache
	{
		private final Map<String, Tx> txCache = new HashMap<String, Tx> ();

		private final Map<String, BigInteger> outputs = new HashMap<String, BigInteger> ();

		public void remove (String key, BigInteger c)
		{
			BigInteger mask = outputs.get (key);
			if ( mask != null )
			{
				mask = mask.xor (c);
				outputs.put (key, mask);

				if ( mask.equals (BigInteger.ZERO) )
				{
					txCache.remove (key);
				}
			}
		}

		public void add (String key, BigInteger c)
		{
			BigInteger mask = outputs.get (key);
			if ( mask == null )
			{
				mask = BigInteger.ZERO;
			}
			outputs.put (key, mask.or (c));
		}

		public List<TxOut> get (String txhash)
		{
			List<TxOut> outs = new ArrayList<TxOut> ();
			Tx cached = txCache.get (txhash);
			if ( cached != null )
			{
				for ( TxOut out : cached.getOutputs () )
				{
					if ( contains (txhash, out.getIx ()) )
					{
						outs.add (out);
					}
				}
			}
			return outs;
		}

		public boolean contains (String key, long ix)
		{
			BigInteger mask = outputs.get (key);
			if ( mask == null )
			{
				return false;
			}
			return mask.testBit ((int) ix);
		}

		public void add (Blk b)
		{
			for ( Tx t : b.getTransactions () )
			{
				if ( outputs.containsKey (t.getHash ()) )
				{
					txCache.put (t.getHash (), t.flatCopy ());
				}
				for ( TxIn in : t.getInputs () )
				{
					if ( !outputs.containsKey (in.getSourceHash ()) )
					{
						txCache.remove (in.getSourceHash ());
					}
				}
			}
		}

		public void remove (Blk b)
		{
			for ( Tx t : b.getTransactions () )
			{
				if ( !outputs.containsKey (t.getHash ()) )
				{
					txCache.remove (t.getHash ());
				}
				for ( TxIn in : t.getInputs () )
				{
					if ( outputs.containsKey (in.getSourceHash ()) )
					{
						txCache.put (in.getSourceHash (), in.getSource ().getTransaction ().flatCopy ());
					}
				}
			}
		}

		public void cacheUTXO ()
		{
			if ( !outputs.keySet ().isEmpty () )
			{
				QTx tx = QTx.tx;
				List<String> txs = new ArrayList<String> ();
				txs.addAll (outputs.keySet ());
				log.trace ("... total " + txs.size ());
				JPAQuery q = new JPAQuery (entityManager);
				int n = 0;
				for ( Tx t : q.from (tx).where (tx.hash.in (txs)).list (tx) )
				{
					txCache.put (t.getHash (), t.flatCopy ());
					entityManager.detach (t);
					if ( ++n % 10000 == 0 )
					{
						log.trace ("... read " + n);
					}
				}
			}
		}
	}

	private static class CachedHead
	{
		private Long id;
		private CachedBlock last;
		private double chainWork;
		private long height;
		private CachedHead previous;
		private final Set<CachedBlock> blocks = new HashSet<CachedBlock> ();

{'fraction_non_alphanumeric': 0.1149748572402625, 'fraction_numerical': 0.004091025313219126, 'mean_word_length': 3.4837600305693543, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4740022', 'n_tokens_mistral': 4311, 'n_tokens_neox': 3684, 'n_words': 961}