<commit_before>import logging
import time
import struct
from dispatch import receiver

from stoppable import StoppableLoopThread
import signals
from db import DB
import utils
import rlp
import blocks

logger = logging.getLogger(__name__)

rlp_hash_hex = lambda data: utils.sha3(rlp.encode(data)).encode('hex')


class ChainManager(StoppableLoopThread):

    """
    Manages the chain and requests to it.
    """

    coinbase = "0" * 40  # FIXME

    def __init__(self):
        super(ChainManager, self).__init__()
        self.transactions = set()
        self.blockchain = DB(utils.get_db_path())
        logger.debug('Chain @ #%d %s' %
                     (self.head.number, self.head.hex_hash()))
        self._mining_nonce = 0

    def configure(self, config):
        self.config = config

    @property
    def head(self):
        if not 'HEAD' in self.blockchain:
            self._initialize_blockchain()
        ptr = self.blockchain.get('HEAD')
        return blocks.Block.deserialize(self.blockchain.get(ptr))

    def _update_head(self, block):
        self.blockchain.put('HEAD', block.hash())
        self._mining_nonce = 0  # reset

    def get(self, blockhash):
        return blocks.get_block(blockhash)

    def has_block(self, blockhash):
        return blockhash in self.blockchain

    def __contains__(self, blockhash):
        return self.has_block(blockhash)

    def _store_block(self, block):
        self.blockchain.put(block.hash(), block.serialize())

    def _initialize_blockchain(self):
        logger.info('Initializing new chain @ %s', utils.get_db_path())
        genesis = blocks.genesis()
        self._store_block(genesis)
        self._update_head(genesis)
        self.blockchain.commit()

    # Returns True if block is latest
    def add_block(self, block):

        def summarized_difficulty(block):
            # calculate the summarized_difficulty (on the fly for now)
            if block.hash() == blocks.GENESIS_HASH:
                return block.difficulty
            else:
                return block.difficulty + summarized_difficulty(block.get_parent())

        # make sure we know the parent
        if not block.has_parent() and block.hash() != blocks.GENESIS_HASH:
            logger.debug('Missing parent for block %r' % block.hex_hash())
            return False  # FIXME

        self._store_block(block)

        # set to head if this makes the longest chain w/ most work
        if summarized_difficulty(block) > summarized_difficulty(self.head):
            self._update_head(block)
            return True

        self.blockchain.commit()
        return False

    def synchronize_blockchain(self):
        # FIXME: execute once, when connected to required num peers
        signals.remote_chain_requested.send(
            sender=self, parents=[self.head.hash()], count=30)

    def loop_body(self):
        self.mine()
        time.sleep(.01)
        # self.synchronize_blockchain()

    def mine(self, steps=1000):
        """
        It is formally defined as PoW:
        PoW(H, n) = BE(SHA3(SHA3(RLP(Hn)) o n))
        where: RLP(Hn) is the RLP encoding of the block header H,
        not including the final nonce component; SHA3 is the SHA3 hash function accepting
        an arbitrary length series of bytes and evaluating to a series of 32 bytes
        (i.e. 256-bit); n is the nonce, a series of 32 bytes; o is the series concatenation
        operator; BE(X) evaluates to the value equal to X when interpreted as a 
        big-endian-encoded integer.
        """

        pack = struct.pack
        sha3 = utils.sha3
        beti = utils.big_endian_to_int

        nonce = self._mining_nonce
        block = blocks.Block.init_from_parent(
            self.head, coinbase=self.coinbase)
        if nonce == 0:
            logger.debug('Committing to mine on %s', block.hex_hash())

        nonce_bin_prefix = '\x00' * (32 - len(pack('>q', 0)))
        prefix = block.serialize_header_without_nonce() + nonce_bin_prefix

        target = 2 ** 256 / block.difficulty

        for nonce in range(nonce, nonce + steps):
            h = sha3(sha3(prefix + pack('>q', nonce)))
            l256 = beti(h)
            if l256 < target:
                block.nonce = nonce_bin_prefix + pack('q', nonce)
                assert len(block.nonce) == 32
                logger.debug('Nonce found %d %r', nonce, block.nonce)
                time.sleep(1)
                # create new block
                self.add_block(block)<commit_msg>fix:big-endian nonce<commit_after>import logging
import time
import struct
from dispatch import receiver

from stoppable import StoppableLoopThread
import signals
from db import DB
import utils
import rlp
import blocks

logger = logging.getLogger(__name__)

rlp_hash_hex = lambda data: utils.sha3(rlp.encode(data)).encode('hex')


class ChainManager(StoppableLoopThread):

    """
    Manages the chain and requests to it.
    """

    coinbase = "0" * 40  # FIXME

    def __init__(self):
        super(ChainManager, self).__init__()
        self.transactions = set()
        self.blockchain = DB(utils.get_db_path())
        logger.debug('Chain @ #%d %s' %
                     (self.head.number, self.head.hex_hash()))
        self._mining_nonce = 0

    def configure(self, config):
        self.config = config

    @property
    def head(self):
        if not 'HEAD' in self.blockchain:
            self._initialize_blockchain()
        ptr = self.blockchain.get('HEAD')
        return blocks.Block.deserialize(self.blockchain.get(ptr))

    def _update_head(self, block):
        self.blockchain.put('HEAD', block.hash())
        self._mining_nonce = 0  # reset

    def get(self, blockhash):
        return blocks.get_block(blockhash)

    def has_block(self, blockhash):
        return blockhash in self.blockchain

    def __contains__(self, blockhash):
        return self.has_block(blockhash)

    def _store_block(self, block):
        self.blockchain.put(block.hash(), block.serialize())

    def _initialize_blockchain(self):
        logger.info('Initializing new chain @ %s', utils.get_db_path())
        genesis = blocks.genesis()
        self._store_block(genesis)
        self._update_head(genesis)
        self.blockchain.commit()

    # Returns True if block is latest
    def add_block(self, block):

        def summarized_difficulty(block):
            # calculate the summarized_difficulty (on the fly for now)
            if block.hash() == blocks.GENESIS_HASH:
                return block.difficulty
            else:
                return block.difficulty + summarized_difficulty(block.get_parent())

        # make sure we know the parent
        if not block.has_parent() and block.hash() != blocks.GENESIS_HASH:
            logger.debug('Missing parent for block %r' % block.hex_hash())
            return False  # FIXME

        self._store_block(block)

        # set to head if this makes the longest chain w/ most work
        if summarized_difficulty(block) > summarized_difficulty(self.head):
            self._update_head(block)
            return True

        self.blockchain.commit()
        return False

    def synchronize_blockchain(self):
        # FIXME: execute once, when connected to required num peers
        signals.remote_chain_requested.send(
            sender=self, parents=[self.head.hash()], count=30)

    def loop_body(self):
        self.mine()
        time.sleep(.01)
        # self.synchronize_blockchain()

    def mine(self, steps=1000):
        """
        It is formally defined as PoW:
        PoW(H, n) = BE(SHA3(SHA3(RLP(Hn)) o n))
        where: RLP(Hn) is the RLP encoding of the block header H,
        not including the final nonce component; SHA3 is the SHA3 hash function accepting
        an arbitrary length series of bytes and evaluating to a series of 32 bytes
        (i.e. 256-bit); n is the nonce, a series of 32 bytes; o is the series concatenation
        operator; BE(X) evaluates to the value equal to X when interpreted as a 
        big-endian-encoded integer.
        """

        pack = struct.pack
        sha3 = utils.sha3
        beti = utils.big_endian_to_int

        nonce = self._mining_nonce
        block = blocks.Block.init_from_parent(
            self.head, coinbase=self.coinbase)
        if nonce == 0:
            logger.debug('Mining %s', block.hex_hash())
            logger.debug('Difficulty %s', block.difficulty)

        nonce_bin_prefix = '\x00' * (32 - len(pack('>q', 0)))
        prefix = block.serialize_header_without_nonce() + nonce_bin_prefix

        target = 2 ** 256 / block.difficulty

        for nonce in range(nonce, nonce + steps):
            h = sha3(sha3(prefix + pack('>q', nonce)))
            l256 = beti(h)
            if l256 < target:
                block.nonce = nonce_bin_prefix + pack('q', nonce)
                assert len(block.nonce) == 32
                logger.debug('Nonce found %d %r', nonce, block.nonce)
                time.sleep(1)
                # create new block
                self.add_block(block)
{'fraction_non_alphanumeric': 0.09841129744042365, 'fraction_numerical': 0.01059135039717564, 'mean_word_length': 2.3144424131627055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5851390', 'n_tokens_mistral': 2773, 'n_tokens_neox': 2610, 'n_words': 720}