<commit_before>package org.ethereum.trie;

import org.ethereum.datasource.inmem.HashMapDB;
import org.ethereum.datasource.Source;
import org.ethereum.util.FastByteComparisons;
import org.ethereum.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spongycastle.util.encoders.Hex;

import java.util.Arrays;
import java.util.List;

import static java.util.Arrays.copyOfRange;
import static org.ethereum.crypto.HashUtil.EMPTY_TRIE_HASH;
import static org.ethereum.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.ethereum.util.ByteUtil.matchingNibbleLength;
import static org.ethereum.util.CompactEncoder.*;
import static org.spongycastle.util.Arrays.concatenate;

/**
 * Created by <NAME> on 05.10.2016.
 */
public class TrieImpl implements Trie<byte[]> {


    private static final Logger logger = LoggerFactory.getLogger("trie");

    private static byte PAIR_SIZE = 2;
    private static byte LIST_SIZE = 17;

    Source<byte[], Value> cache;
    Object  root;

    public TrieImpl() {
        this((byte[]) null);
    }

    public TrieImpl(byte[] root) {
        this(new HashMapDB<Value>(), root);
    }

    public TrieImpl(Source<byte[], Value> cache) {
        this(cache, null);
    }

    public TrieImpl(Source<byte[], Value> cache, byte[] root) {
        this.cache = cache;
        if (root != null && FastByteComparisons.equal(root, EMPTY_TRIE_HASH)) {
            root = null;
        }
        this.root = root;
    }

    public TrieIterator getIterator() {
        throw new RuntimeException("Not supported");
    }

    public Object getRoot() {
        return root;
    }

    public void setRoot(byte[] root) {
        this.root = root;
    }

    /**************************************
     * Public (query) interface functions *
     **************************************/

    /**
     * Retrieve a value from a key as String.
     */
    public byte[] get(String key) {
        return this.get(key.getBytes());
    }

    @Override
    public byte[] get(byte[] key) {
        synchronized (cache) {
            if (logger.isDebugEnabled())
                logger.debug("Retrieving key {}", Hex.toHexString(key));
            byte[] k = binToNibbles(key);
            Value c = new Value(this.get(this.root, k));

            return c.asBytes();
        }
    }

    /**
     * Insert key/value pair into trie.
     */
    @Override
    public void put(byte[] key, byte[] value) {
        if (key == null)
            throw new NullPointerException("Key should not be blank");
        synchronized (cache) {
            byte[] k = binToNibbles(key);

            this.root = this.insertOrDelete(this.root, k, value);
            if (logger.isDebugEnabled()) {
                logger.debug("Added key {} and value {}", Hex.toHexString(key), Hex.toHexString(value));
                logger.debug("New root-hash: {}", Hex.toHexString(this.getRootHash()));
            }
        }
    }

    /**
     * Delete a key/value pair from the trie.
     */
    public void delete(String key) {
        this.put(key.getBytes(), EMPTY_BYTE_ARRAY);
    }

    @Override
    public void delete(byte[] key) {
        synchronized (cache) {
            this.put(key, EMPTY_BYTE_ARRAY);
            if (logger.isDebugEnabled()) {
                logger.debug("Deleted value for key {}", Hex.toHexString(key));
                logger.debug("New root-hash: {}", Hex.toHexString(this.getRootHash()));
            }
        }
    }

    @Override
    public byte[] getRootHash() {
        synchronized (cache) {
            if (root == null
                    || (root instanceof byte[] && ((byte[]) root).length == 0)
                    || (root instanceof String && "".equals(root))) {
                return EMPTY_TRIE_HASH;
            } else if (root instanceof byte[]) {
                return (byte[]) this.getRoot();
            } else {
                Value rootValue = new Value(this.getRoot());
                return rootValue.hash();
            }
        }
    }

    /****************************************
     *          Private functions           *
     ****************************************/

    private Object get(Object node, byte[] key) {
        synchronized (cache) {

            // Return the node if key is empty (= found)
            if (key.length == 0 || isEmptyNode(node)) {
                return node;
            }

            Value currentNode = this.getNode(node);
            if (currentNode == null) return null;

            if (currentNode.length() == PAIR_SIZE) {
                // Decode the key
                byte[] k = unpackToNibbles(currentNode.get(0).asBytes());
                Object v = currentNode.get(1).asObj();

                if (key.length >= k.length && Arrays.equals(k, copyOfRange(key, 0, k.length))) {
                    return this.get(v, copyOfRange(key, k.length, key.length));
                } else {
                    return "";
                }
            } else {
                return this.get(currentNode.get(key[0]).asObj(), copyOfRange(key, 1, key.length));
            }
        }
    }

    private Object insertOrDelete(Object node, byte[] key, byte[] value) {
        if (value.length != 0) {
            return this.insert(node, key, value);
        } else {
            return this.delete(node, key);
        }
    }

    /**
     * Update or add the item inside a node.
     * @return the updated node with rlp encoded
     */
    private Object insert(Object node, byte[] key, Object value) {<commit_msg>Fail fast on Trie.get() when db is corrupted<commit_after>package org.ethereum.trie;

import org.ethereum.datasource.inmem.HashMapDB;
import org.ethereum.datasource.Source;
import org.ethereum.util.FastByteComparisons;
import org.ethereum.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spongycastle.util.encoders.Hex;

import java.util.Arrays;
import java.util.List;

import static java.util.Arrays.copyOfRange;
import static org.ethereum.crypto.HashUtil.EMPTY_TRIE_HASH;
import static org.ethereum.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.ethereum.util.ByteUtil.matchingNibbleLength;
import static org.ethereum.util.CompactEncoder.*;
import static org.spongycastle.util.Arrays.concatenate;

/**
 * Created by <NAME> on 05.10.2016.
 */
public class TrieImpl implements Trie<byte[]> {


    private static final Logger logger = LoggerFactory.getLogger("trie");

    private static byte PAIR_SIZE = 2;
    private static byte LIST_SIZE = 17;

    Source<byte[], Value> cache;
    Object  root;

    public TrieImpl() {
        this((byte[]) null);
    }

    public TrieImpl(byte[] root) {
        this(new HashMapDB<Value>(), root);
    }

    public TrieImpl(Source<byte[], Value> cache) {
        this(cache, null);
    }

    public TrieImpl(Source<byte[], Value> cache, byte[] root) {
        this.cache = cache;
        if (root != null && FastByteComparisons.equal(root, EMPTY_TRIE_HASH)) {
            root = null;
        }
        this.root = root;
    }

    public TrieIterator getIterator() {
        throw new RuntimeException("Not supported");
    }

    public Object getRoot() {
        return root;
    }

    public void setRoot(byte[] root) {
        this.root = root;
    }

    /**************************************
     * Public (query) interface functions *
     **************************************/

    /**
     * Retrieve a value from a key as String.
     */
    public byte[] get(String key) {
        return this.get(key.getBytes());
    }

    @Override
    public byte[] get(byte[] key) {
        synchronized (cache) {
            if (logger.isDebugEnabled())
                logger.debug("Retrieving key {}", Hex.toHexString(key));
            byte[] k = binToNibbles(key);
            Value c = new Value(this.get(this.root, k));

            return c.asBytes();
        }
    }

    /**
     * Insert key/value pair into trie.
     */
    @Override
    public void put(byte[] key, byte[] value) {
        if (key == null)
            throw new NullPointerException("Key should not be blank");
        synchronized (cache) {
            byte[] k = binToNibbles(key);

            this.root = this.insertOrDelete(this.root, k, value);
            if (logger.isDebugEnabled()) {
                logger.debug("Added key {} and value {}", Hex.toHexString(key), Hex.toHexString(value));
                logger.debug("New root-hash: {}", Hex.toHexString(this.getRootHash()));
            }
        }
    }

    /**
     * Delete a key/value pair from the trie.
     */
    public void delete(String key) {
        this.put(key.getBytes(), EMPTY_BYTE_ARRAY);
    }

    @Override
    public void delete(byte[] key) {
        synchronized (cache) {
            this.put(key, EMPTY_BYTE_ARRAY);
            if (logger.isDebugEnabled()) {
                logger.debug("Deleted value for key {}", Hex.toHexString(key));
                logger.debug("New root-hash: {}", Hex.toHexString(this.getRootHash()));
            }
        }
    }

    @Override
    public byte[] getRootHash() {
        synchronized (cache) {
            if (root == null
                    || (root instanceof byte[] && ((byte[]) root).length == 0)
                    || (root instanceof String && "".equals(root))) {
                return EMPTY_TRIE_HASH;
            } else if (root instanceof byte[]) {
                return (byte[]) this.getRoot();
            } else {
                Value rootValue = new Value(this.getRoot());
                return rootValue.hash();
            }
        }
    }

    /****************************************
     *          Private functions           *
     ****************************************/

    private Object get(Object node, byte[] key) {
        synchronized (cache) {

            // Return the node if key is empty (= found)
            if (key.length == 0 || isEmptyNode(node)) {
                return node;
            }

            Value currentNode = this.getNode(node);
            if (currentNode == null) {
                throw new RuntimeException("Invalid Trie state, missing node " + new Value(node));
            }

            if (currentNode.length() == PAIR_SIZE) {
                // Decode the key
                byte[] k = unpackToNibbles(currentNode.get(0).asBytes());
                Object v = currentNode.get(1).asObj();

                if (key.length >= k.length && Arrays.equals(k, copyOfRange(key, 0, k.length))) {
                    return this.get(v, copyOfRange(key, k.length, key.length));
                } else {
                    return "";
                }
            } else {
                return this.get(currentNode.get(key[0]).asObj(), copyOfRange(key, 1, key.length));
            }
        }
    }

    private Object insertOrDelete(Object node, byte[] key, byte[] value) {
        if (value.length != 0) {
            return this.insert(node, key, value);
        } else {
            return this.delete(node, key);
        }
    }

    /**
     * Update or add the item inside a node.
     * @return the updated node with rlp encoded
     */
    private Object insert(Object node, byte[] key, Object value) {
{'fraction_non_alphanumeric': 0.1504598624877221, 'fraction_numerical': 0.003750334851326011, 'mean_word_length': 2.0172413793103448, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2367683', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3181, 'n_words': 766}