<commit_before># Distributed under the MIT software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.

# -features POS, quantum secure signature scheme..
from qrlcore import logger
from qrlcore.block import Block

__author__ = 'pete'
import json
import random
import struct
import time
from collections import Counter, defaultdict
from copy import deepcopy
from decimal import Decimal

from twisted.internet import reactor
from twisted.internet.protocol import ServerFactory, Protocol

import fork
import helper
from messagereceipt import MessageReceipt
from qrlcore.merkle import sha256
from qrlcore.transaction import StakeTransaction, SimpleTransaction
import configuration as config


class NodeState:
    def __init__(self):
        self.state = 'unsynced'
        self.epoch_diff = -1


class POS:
    def __init__(self, chain, p2pFactory, nodeState, ntp):
        self.master_mr = MessageReceipt()
        self.nodeState = nodeState
        self.ntp = ntp
        self.chain = chain
        self.r1_time_diff = defaultdict(list)
        self.r2_time_diff = defaultdict(list)

        self.incoming_blocks = {}
        self.last_pos_cycle = 0
        self.last_selected_height = 0
        self.last_bk_time = 0
        self.last_pb_time = 0
        self.next_header_hash = None
        self.next_block_number = None
        self.fmbh_allowed_peers = {}
        self.fmbh_blockhash_peers = {}

        self.p2pFactory = p2pFactory

    def update_node_state(self, state):
        self.nodeState.state = state
        logger.info(('Status changed to ', self.nodeState.state))
        if self.nodeState.state == 'synced':
            self.nodeState.epoch_diff = 0
            self.last_pos_cycle = time.time()
            self.restart_post_block_logic()
        elif self.nodeState.state == 'unsynced':
            self.last_bk_time = time.time()
            self.restart_unsynced_logic()
        elif self.nodeState.state == 'forked':
            self.stop_post_block_logic()
        elif self.nodeState.state == 'syncing':
            self.last_pb_time = time.time()

    def stop_monitor_bk(self):
        try:
            reactor.monitor_bk.cancel()
        except:
            pass

    def restart_monitor_bk(self, delay=60):
        self.stop_monitor_bk()
        reactor.monitor_bk = reactor.callLater(delay, self.monitor_bk)

    def monitor_bk(self):
        if (self.nodeState.state == 'synced' or self.nodeState.state == 'unsynced') and 90 < time.time() - self.last_pos_cycle:
            if self.nodeState.state == 'synced':
                self.stop_post_block_logic()
                self.reset_everything()
                self.update_node_state('unsynced')
                self.epoch_diff = -1
            elif time.time() - self.last_bk_time > 120:
                self.last_pos_cycle = time.time()
                logger.info(' POS cycle activated by monitor_bk() ')
                self.update_node_state('synced')

        if self.nodeState.state == 'syncing' and time.time() - self.last_pb_time > 60:
            self.stop_post_block_logic()
            self.reset_everything()
            self.update_node_state('unsynced')
            self.epoch_diff = -1
        reactor.monitor_bk = reactor.callLater(60, self.monitor_bk)

    def peers_blockheight_headerhash(self):
        for peer in self.p2pFactory.peers:
            peer.fetch_headerhash_n(self.chain.m_blockheight())

    def check_fork_status(self):
        current_height = self.chain.m_blockheight()
        block_hash_counter = Counter()
        for peer in self.p2pFactory.peers:
            if current_height in peer.blocknumber_headerhash.keys():
                block_hash_counter[peer.blocknumber_headerhash[current_height]] += 1

        blockhash = block_hash_counter.most_common(1)
        if blockhash:
            blockhash = blockhash[0][0]
            actual_blockhash = self.chain.m_get_block(current_height).blockheader.headerhash
            if actual_blockhash != blockhash:
                logger.info('Blockhash didnt matched in peers_blockheight()')
                logger.info(('Local blockhash - ', actual_blockhash))
                logger.info(('Consensus blockhash - ', blockhash))
                fork.fork_recovery(current_height, self.chain, self.randomize_headerhash_fetch)
                return True
        return

    def peers_blockheight(self):
        if self.nodeState.state == 'syncing':
            return
        if self.check_fork_status():
            return

        block_height_counter = Counter()

        for peer in self.p2pFactory.peers:
            block_height_counter[peer.blockheight] += 1

        blocknumber = block_height_counter.most_common(1)
        if not blocknumber:
            return  # TODO : Re-Schedule with delay

        blocknumber = blocknumber[0][0]

        if blocknumber > self.chain.height():  # chain.m_blockheight():  len(chain.m_blockchain)
            # pending_blocks['target'] = blocknumber
            logger.info(('Calling downloader from peers_blockheight due to no POS CYCLE ', blocknumber))
            logger.info(('Download block from ', self.chain.height() + 1, ' to ', blocknumber))
            self.last_pb_time = time.time()
            self.update_node_state('syncing')
            self.randomize_block_fetch(self.chain.height() + 1)
        return

    def schedule_peers_blockheight(self, delay=100):
        try:
            reactor.peers_blockheight.cancel()
        except Exception:
            pass
        reactor.peers_blockheight = reactor.callLater(delay, self.peers_blockheight)
        try:
            reactor.peers_blockheight_headerhash.cancel()
        except Exception:
            pass
        reactor.peers_blockheight_headerhash = reactor.callLater(70, self.peers_blockheight_headerhash)

    # pos functions. an asynchronous loop.

    # first block 1 is created with the stake list for epoch 0 decided from circulated st transactions

    def pre_pos_1(self, data=None):  # triggered after genesis for block 1..
        logger.info(('pre_pos_1'))
        # are we a staker in the stake list?

        if self.chain.mining_address in self.chain.m_blockchain[0].stake_list:
            logger.info(('mining address:', self.chain.mining_address, ' in the genesis.stake_list'))

            self.chain.my[0][1].hashchain(epoch=0)
            self.chain.hash_chain = self.chain.my[0][1].hc
            self.chain.block_chain_buffer.hash_chain[0] = self.chain.my[0][1].hc

            logger.info(('hashchain terminator: ', self.chain.my[0][1].hc_terminator))
            st = StakeTransaction().create_stake_transaction(mining_address=self.chain.mining_address,
                                                             blocknumber=0,
                                                             data=self.chain.my[0][1],
                                                             hashchain_terminator=self.chain.my[0][1].hc_terminator,
                                                             first_hash=self.chain.my[0][1].hc[-1][-2],
                                                             balance=self.chain.state.state_balance(self.chain.mining_address))
            self.chain.wallet.f_save_winfo()
            self.chain.add_st_to_pool(st)
            # send the stake tx to generate hashchain terminators for the staker addresses..
            self.p2pFactory.send_st_to_peers(st)
            logger.info('await delayed call to build staker list from genesis')
            reactor.callLater(5, self.pre_pos_2, st)
            return

        logger.info('not in stake list..no further pre_pos_x calls')
        return

    def pre_pos_2(self, data=None):
        logger.info(('pre_pos_2'))
        if self.chain.height() >= 1:
            return
        # assign hash terminators to addresses and generate a temporary stake list ordered by st.hash..

        tmp_list = []

        for st in self.chain.stake_pool:
            if st.txfrom in self.chain.m_blockchain[0].stake_list:
                tmp_list.append([st.txfrom, st.hash, 0, st.first_hash, self.chain.state.state_balance(st.txfrom)])

        # required as doing chain.stake_list.index(s) which will result into different number on different server
        self.chain.block_chain_buffer.epoch_seed = self.chain.state.calc_seed(tmp_list)
        self.chain.stake_list = sorted(tmp_list,
                                       key=lambda staker: self.chain.score(stake_address=staker[0],
                                                                           reveal_one=sha256(str(staker[1])),
                                                                           balance=self.chain.state.state_balance(st.txfrom),
                                                                           seed=self.chain.block_chain_buffer.epoch_seed))

        logger.info(('genesis stakers ready = ', len(self.chain.stake_list), '/', config.dev.minimum_required_stakers))
        logger.info(('node address:', self.chain.mining_address))

        if len(self.chain.stake_list) < config.dev.minimum_required_stakers:  # stake pool still not full..reloop..
            self.p2pFactory.send_st_to_peers(data)
            logger.info('waiting for stakers.. retry in 5s')
            reactor.callID = reactor.callLater(5, self.pre_pos_2, data)
            return
        logger.info((str(self.chain.stake_list)))

        if self.chain.mining_address == self.chain.stake_list[0][0]:
            logger.info('designated to create block 1: building block..')

            # create the genesis block 2 here..
            my_hash_chain, _ = self.chain.select_hashchain(self.chain.m_blockchain[-1].blockheader.headerhash,
                                                           self.chain.mining_address, self.chain.my[0][1].hc,
                                                           blocknumber=1)
            b = self.chain.m_create_block(my_hash_chain[-2])
            self.pre_block_logic(b)
        else:
            logger.info(('await block creation by stake validator:', self.chain.stake_list[0][0]))
            self.last_bk_time = time.time()
            self.restart_unsynced_logic()
        return

    def process_transactions(self, num):
        tmp_num = num
        for tx in self.chain.pending_tx_pool:
            tmp_num -= 1
            tx_peer = tx[1]
            tx = tx[0]
            if tx.validate_tx() != True:
                logger.info(('>>>TX ', tx.txhash, 'failed validate_tx'))
                continue

            isValidState = tx.state_validate_tx(
                state=self.chain.state,
                transaction_pool=self.chain.transaction_pool
            )
            if not isValidState:
                logger.info(('>>>TX', tx.txhash, 'failed state_validate'))
                continue

            logger.info(('>>>TX - ', tx.txhash, ' from - ', tx_peer.transport.getPeer().host, ' relaying..'))
            self.chain.add_tx_to_pool(tx)

            txn_msg = tx_peer.wrap_message('TX', tx.transaction_to_json())
            for peer in tx_peer.factory.peers:
                if peer != tx_peer:
                    peer.transport.write(txn_msg)

        for i in range(num - tmp_num):
            del self.chain.pending_tx_pool[0]
            del self.chain.pending_tx_pool_hash[0]

    # create new block..

    def create_new_block(self, winner, reveals, vote_hashes, last_block_number):
        logger.info(('create_new_block #', (last_block_number + 1)))
        tx_list = []
        for t in self.chain.transaction_pool:
            tx_list.append(t.txhash)
        block_obj = self.chain.create_stake_block(tx_list, winner, reveals, vote_hashes, last_block_number)

        return block_obj

    def reset_everything(self, data=None):
        logger.info('** resetting loops and emptying chain.stake_reveal_one and chain.expected_winner ')
        for r in self.chain.stake_reveal_one:
            msg_hash = r[5]
            self.master_mr.deregister(msg_hash, 'R1')

        del self.chain.stake_reveal_one[:]
        return

    def filter_reveal_one_two(self, blocknumber=None):
        if not blocknumber:
            blocknumber = self.chain.m_blockchain[-1].blockheader.blocknumber

        self.chain.stake_reveal_one = filter(lambda s: s[2] > blocknumber,
                                             self.chain.stake_reveal_one)

        return

    # TODO: Incomplete fn, use to select the maximum blockheight by consensus
    def select_blockheight_by_consensus(self):
        block_height_counter = Counter()
        # for identity in self.fmbh_allowed_peers:
        #    block_height_counter[s[2]] += 1
        target_block_height = block_height_counter.most_common(1)

        if len(target_block_height) == 0:
            return None

        last_selected_height = target_block_height[0][0]
        return last_selected_height

    '''
    Unsynced Logic
    1.	Request for maximum blockheight and passes bock number X
    2.	Peers response chain height with headerhash and the headerhash of block number X
    3.	Unsynced node, selects most common chain height, matches the headerhash of block number X
    4.	If headerhash of block number X doesn't match, change state to Forked
    5.	If headerhash of block number X matches, perform Downloading of blocks from those selected peers
    '''

    def restart_unsynced_logic(self, delay=0):
        try:
            reactor.unsynced_logic.cancel()
        except:
            pass
        reactor.unsynced_logic = reactor.callLater(delay, self.unsynced_logic)

    def unsynced_logic(self):
        if self.nodeState.state == 'synced':
            return

        self.fmbh_blockhash_peers = {}
        self.fmbh_allowed_peers = {}
        for peer in self.p2pFactory.peers:
            self.fmbh_allowed_peers[peer.identity] = None
            peer.fetch_FMBH()
        reactor.unsynced_logic = reactor.callLater(20, self.start_download)

    def start_download(self):
        # add peers and their identity to requested list
        # FMBH
        if self.nodeState.state == 'synced':
            return
        logger.info(('Checking Download..'))
        '''
        global fmbh_blockhash_peers
        max_height = None
        selected_blockhash = None
        for blockheaderhash in fmbh_blockhash_peers:
            if fmbh_blockhash_peers[blockheaderhash]['blocknumber']>max_height:
                max_height = fmbh_blockhash_peers[blockheaderhash]['blocknumber']
                selected_blockhash = blockheaderhash
        for peer in fmbh_blockhash_peers[selected_blockhash]['peers']:
            f.target_peers = {}
            f.target_peers[peer.identity] = peer
        
        if max_height == None or max_height<=chain.height():
            chain.state.update('synced')
            return
        
        chain.state.update('syncing')
        pending_blocks['start_block'] = chain.m_blockchain[-1].blockheader.blocknumber
        pending_blocks['target'] = fmbh_blockhash_peers[selected_blockhash]['blocknumber']
        pending_blocks['headerhash'] = selected_blockhash
        randomize_block_fetch(chain.height() + 1)
        '''
        max = -1
        max_headerhash = None
        for headerhash in self.fmbh_blockhash_peers:
            if self.fmbh_blockhash_peers[headerhash]['blocknumber'] > self.chain.height():
                if len(self.fmbh_blockhash_peers[headerhash]['peers']) > max:
                    max = len(self.fmbh_blockhash_peers[headerhash]['peers'])
                    max_headerhash = headerhash

        # Adding all peers
        # TODO only trusted peer
        # for peer in self.p2pFactory.peers:
        if not max_headerhash:
            logger.info('No peers responded FMBH request')
            return
        for peer in self.fmbh_blockhash_peers[max_headerhash]['peers']:
            self.p2pFactory.target_peers[peer.identity] = peer
        self.update_node_state('syncing')
        self.randomize_block_fetch(self.chain.height() + 1)

    def pre_block_logic(self, block, peer_identity=None):
        if len(self.chain.m_blockchain) == 0:
            self.chain.m_read_chain()

        blocknumber = block.blockheader.blocknumber
        headerhash = block.blockheader.headerhash
        prev_blockheaderhash = block.blockheader.prev_blockheaderhash
        curr_epoch = self.chain.height() /config.dev.blocks_per_epoch
        next_epoch = (self.chain.height() + 1) /config.dev.blocks_per_epoch
        chain_buffer_height = self.chain.block_chain_buffer.height()

        if blocknumber <= self.chain.height():
            return False

        if self.nodeState.state == 'synced':
            if self.chain.block_chain_buffer.add_block(block):
                self.p2pFactory.send_block_to_peers(block, peer_identity)
        else:
            if chain_buffer_height + 1 == blocknumber:
                if blocknumber > 1 and self.chain.block_chain_buffer.add_block(block):
                    self.p2pFactory.send_block_to_peers(block, peer_identity)
                elif blocknumber == 1 and self.chain.block_chain_buffer.add_block_mainchain(block):
                    self.p2pFactory.send_block_to_peers(block, peer_identity)
                self.update_node_state('synced')
            else:
                self.chain.block_chain_buffer.add_pending_block(block)

        if self.nodeState.state == 'synced':
            if chain_buffer_height + 1 == blocknumber:
                self.last_pos_cycle = time.time()
                block_timestamp = int(block.blockheader.timestamp)
                curr_time = int(self.ntp.getTime())
                delay = config.dev.POS_delay_after_block - min(c.POS_delay_after_block, max(0, curr_time - block_timestamp))

                self.restart_post_block_logic(delay)
        # commented
        '''
        if chain.m_blockchain and block.blockheader.epoch == curr_epoch or block.blockheader.epoch == next_epoch:
            if blocknumber < chain.height() + 1 and headerhash != chain.m_blockchain[blocknumber].blockheader.headerhash:
                if not chain.validate_block(block):
                    return False

                sum_block_reward_old_sl, sum_block_reward_new_sl = sumBlockReward(chain.m_blockchain[block.blockheader.blocknumber].blockheader.stake_selector, block.blockheader.stake_selector, block.blockheader.blocknumber)

                #Compare if strongest, then replace from the buffer
                total_stakers_new_block, score_new_block = getBlockStakeInfo(block, block_reward=sum_block_reward_new_sl)
                total_stakers, score = getBlockStakeInfo(chain.m_blockchain[blocknumber], block_reward=sum_block_reward_old_sl)
                printL (( 'New Block # ', blocknumber, ' HeaderHash: ', headerhash, ' Score: ', score_new_block, ' sum block reward : ', sum_block_reward_new_sl ))
                printL (( 'Old Block # ', blocknumber, ' HeaderHash: ', chain.m_blockchain[blocknumber].blockheader.headerhash, ' Score: ', score, ' sum block reward : ', sum_block_reward_old_sl ))
                printL (( str(chain.state_balance(block.blockheader.stake_selector)) ))
                if score_new_block < score:  #or total_stakers_new_block>total_stakers#Need to be reviewed
                    tmp_blocks = chain.m_blockchain[blocknumber:][::-1]
                    for tmp_block in tmp_blocks:
                        update_nonce(tmp_block)
                        txn_block_to_pool(tmp_block)
                        del chain.m_blockchain[tmp_block.blockheader.blocknumber]
                    add_block(block)
                    if chain.state.current == 'synced':
                        f.send_block_to_peers(block)

                return True

            if blocknumber == chain.height()+1 and prev_blockheaderhash == chain.m_blockchain[-1].blockheader.headerhash:
                if not chain.validate_block(block):
                    return False

                add_block(block)

                if chain.state.current == 'synced':
                    f.send_block_to_peers(block)
                if chain.state.current == 'unsynced':
                    chain.state.update('synced')
                restart_post_block_logic()
                return True
        '''
        return True

    def stop_post_block_logic(self, delay=0):
        try:
            reactor.post_block_logic.cancel()
            reactor.prepare_winners.cancel()
        except Exception:
            pass

    def restart_post_block_logic(self, delay=0):
        self.stop_post_block_logic()
        reactor.post_block_logic = reactor.callLater(delay,
                                                     self.post_block_logic)

    # post block logic we initiate the next POS cycle, send R1, send ST, reset POS flags and remove unnecessary messages in chain.stake_reveal_one and _two..

    def post_block_logic(self):
        self.filter_reveal_one_two()

        our_reveal = None
        blocknumber = self.chain.block_chain_buffer.height() + 1

        if self.p2pFactory.stake:
            tmp_stake_list = [
                s[0] for s in self.chain.block_chain_buffer.stake_list_get(blocknumber)
            ]
            if self.chain.mining_address in tmp_stake_list:
                our_reveal = self.p2pFactory.send_stake_reveal_one(blocknumber)
                self.schedule_prepare_winners(our_reveal, blocknumber - 1, 30)

            '''tmp_next_stake_list = [
                s[0] for s in self.chain.block_chain_buffer.next_stake_list_get(blocknumber)
            ]'''
            next_stake_list = self.chain.block_chain_buffer.next_stake_list_get(blocknumber)
            next_stake_first_hash = {}
            for s in next_stake_list:
                next_stake_first_hash[s[0]] = s[3]

            epoch = blocknumber // config.dev.blocks_per_epoch
            epoch_blocknum = blocknumber - epoch * config.dev.blocks_per_epoch

            if epoch_blocknum < config.dev.stake_before_x_blocks and self.chain.mining_address not in next_stake_first_hash:
                diff = max(1, ((c.stake_before_x_blocks - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
                if random.randint(1, diff) == 1:
                    self.make_st_tx(blocknumber, None)
            elif epoch_blocknum >= config.dev.stake_before_x_blocks - 1 and self.chain.mining_address in next_stake_first_hash:
                if next_stake_first_hash[self.chain.mining_address] is None:
                    threshold_blocknum = self.chain.state.get_staker_threshold_blocknum(next_stake_list,
                                                                                        self.chain.mining_address)
                    max_threshold_blocknum = config.dev.blocks_per_epoch
                    if threshold_blocknum == config.dev.low_staker_first_hash_block:
                        max_threshold_blocknum = config.dev.high_staker_first_hash_block

                    if epoch_blocknum >= threshold_blocknum - 1 and epoch_blocknum < max_threshold_blocknum - 1:
                        diff = max(1, ((max_threshold_blocknum - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
                        if random.randint(1, diff) == 1:
                            my = deepcopy(self.chain.my[0][1])
                            my.hashchain(epoch=epoch + 1)
                            self.make_st_tx(blocknumber, my.hc[-1][-2])

        return

    def make_st_tx(self, blocknumber, first_hash):
        balance = self.chain.state.state_balance(self.chain.mining_address)
        if balance < config.dev.minimum_staking_balance_required:
            logger.info('Staking not allowed due to insufficient balance')
            logger.info(('Balance ', balance))
            return

        st = StakeTransaction().create_stake_transaction(
            self.chain.mining_address, blocknumber,
            self.chain.my[0][1],
            first_hash=first_hash,
            balance=balance
        )
        self.p2pFactory.send_st_to_peers(st)
        self.chain.wallet.f_save_winfo()
        for num in range(len(self.chain.stake_pool)):
            t = self.chain.stake_pool[num]
            if st.hash == t.hash:
                if st.get_message_hash() == t.get_message_hash():
                    return
                del self.chain.stake_pool[num]
                break

        self.chain.stake_pool.append(st)
        '''
        for t in self.chain.stake_pool:
            if st.hash == t.hash:
                if st.get_message_hash() != t.get_message_hash():
                    t.first_hash = st.first_hash
                return
        self.chain.add_st_to_pool(st)
        '''

    def schedule_prepare_winners(self, our_reveal, last_block_number, delay=0):
        try:
            reactor.prepare_winners.cancel()
        except:
            pass
        reactor.prepare_winners = reactor.callLater(
            delay,
            self.prepare_winners,
            our_reveal=our_reveal,
            last_block_number=last_block_number)

    def prepare_winners(self, our_reveal, last_block_number):
        if not self.nodeState.state == 'synced':
            return
        filtered_reveal_one = []
        reveals = []
        vote_hashes = []
        next_block_num = last_block_number + 1
        for s in self.chain.stake_reveal_one:
            tmp_strongest_headerhash = self.chain.block_chain_buffer.get_strongest_headerhash(last_block_number)
            if s[1] == tmp_strongest_headerhash and s[2] == next_block_num:
                filtered_reveal_one.append(s)
                reveals.append(s[3])
                vote_hashes.append(s[5])

        self.restart_post_block_logic(30)

        if len(filtered_reveal_one) <= 1:
            logger.info(('only received one reveal for this block.. blocknum #', next_block_num))
            return

        epoch = (next_block_num) / config.dev.blocks_per_epoch  # +1 = next block
        seed = self.chain.block_chain_buffer.get_epoch_seed(next_block_num)
        winners = self.chain.select_winners(filtered_reveal_one,
                                            topN=3,
                                            seed=seed)

        # reactor.process_blocks = reactor.callLater(30, process_blocks, winners=winners, our_reveal=our_reveal)

        if not (self.p2pFactory.stake and our_reveal):
            return

        if our_reveal in winners:
            block = self.create_new_block(our_reveal,
                                          reveals,
                                          vote_hashes,
                                          last_block_number)
            self.pre_block_logic(block)  # broadcast this block

        if self.chain.pending_tx_pool:
            if len(self.chain.transaction_pool) < 10:
                logger.info('Processing TXNs if any')
                self.process_transactions(5)

    def randomize_block_fetch(self, blocknumber):
        if self.nodeState.state != 'syncing' or blocknumber <= self.chain.height():
            return

        if len(self.p2pFactory.target_peers.keys()) == 0:
            logger.info(' No target peers found.. stopping download')
            return

        reactor.download_monitor = reactor.callLater(20,
                                                     self.randomize_block_fetch, blocknumber)

        random_peer = self.p2pFactory.target_peers[random.choice(self.p2pFactory.target_peers.keys())]
        random_peer.fetch_block_n(blocknumber)

    def randomize_headerhash_fetch(self, block_number):
        if self.nodeState.state != 'forked':
            return
        if block_number not in fork.pending_blocks or fork.pending_blocks[block_number][1] <= 10:  # retry only 11 times
            headerhash_monitor = reactor.callLater(15, self.randomize_headerhash_fetch, block_number)
            if len(self.p2pFactory.peers) > 0:
                try:
                    if len(self.p2pFactory.fork_target_peers) == 0:
                        for peer in self.p2pFactory.peers:
                            self.p2pFactory.fork_target_peers[peer.identity] = peer
                    if len(self.p2pFactory.fork_target_peers) > 0:
                        random_peer = self.p2pFactory.fork_target_peers[
                            random.choice(
                                self.p2pFactory.fork_target_peers.keys()
                            )
                        ]
                        count = 0
                        if block_number in fork.pending_blocks:
                            count = fork.pending_blocks[block_number][1] + 1
                        fork.pending_blocks[block_number] = [
                            random_peer.identity, count, None, headerhash_monitor
                        ]
                        random_peer.fetch_headerhash_n(block_number)
                except:
                    logger.info('Exception at randomize_headerhash_fetch')
            else:
                logger.info(('No peers connected.. Will try again... randomize_headerhash_fetch: ', block_number))
        else:
            self.update_node_state('unsynced')

    # blockheight map for connected nodes - when the blockheight seems up to date after a sync or error, we check all connected nodes to ensure all on same chain/height..
    # note - may not return correctly during a block propagation..
    # once working alter to identify fork better..

    def blockheight_map(self):

        # i = [block_number, headerhash, self.transport.getPeer().host]

        logger.info(('blockheight_map:'))
        logger.info((self.chain.blockheight_map))

        # first strip out any laggards..
        self.chain.blockheight_map = filter(
            lambda s: s[0] >= self.chain.m_blockheight(),
            self.chain.blockheight_map
        )

        result = True

        # next identify any node entries which are not exactly correct..

        for s in self.chain.blockheight_map:
            if s[0] == self.chain.m_blockheight():
                if s[1] == self.chain.m_blockchain[-1].blockheader.headerhash:
                    logger.info(('node: ', s[2], '@', s[0], 'w/:', s[1], 'OK'))
            elif s[0] > self.chain.m_blockheight():
                logger.info(('warning..', s[2], 'at blockheight', s[0]))
                result = False

        # wipe it..

        del self.chain.blockheight_map[:]

        return result


class P2PProtocol(Protocol):
    def __init__(self):
        self.service = {'reboot': self.reboot,
                        'MR': self.MR,
                        # 'RFM': self.RFM, only for internal usage
                        'SFM': self.SFM,
                        'TX': self.TX,
                        'ST': self.ST,
                        'BM': self.BM,
                        'BK': self.BK,
                        'PBB': self.PBB,
                        'PB': self.PB,
                        'PH': self.PH,
                        'LB': self.LB,
                        'FMBH': self.FMBH,
                        'PMBH': self.PMBH,
                        'MB': self.MB,
                        'CB': self.CB,
                        'BN': self.BN,
                        'FB': self.FB,
                        'FH': self.FH,
                        'PO': self.PO,
                        'PI': self.PI,
                        'PL': self.PL,
                        'RT': self.RT,
                        'PE': self.PE,
                        'VE': self.VE,
                        'R1': self.R1,
                        'IP': self.IP,
                        }
        self.buffer = ''
        self.messages = []
        self.identity = None
        self.blockheight = None
        self.version = ''
        self.blocknumber_headerhash = {}
        self.last_requested_blocknum = None
        self.fetch_tried = 0
        pass

    def parse_msg(self, data):
        try:
            jdata = json.loads(data)
        except:
            return

        func = jdata['type']

        if func not in self.service:
            return

        func = self.service[func]
        try:
            if 'data' in jdata:
                func(jdata['data'])
            else:
                func()
        except:
            logger.info("parse_msg Exception while calling ")
            logger.info(("Func name ", func))
            # logger.info(( "JSON data ", jdata ))
            pass

    def reboot(self, data):
        hash_dict = json.loads(data)
        if not ('hash' in hash_dict and 'nonce' in hash_dict and 'blocknumber' in hash_dict):
            return
        status, error = self.factory.chain.validate_reboot(hash_dict['hash'], hash_dict['nonce'])
        if not status:
            logger.info(('status ', status))
            logger.info(('error ', error))
            return
        for peer in self.factory.peers:
            if peer != self:
                peer.transport.write(self.wrap_message('reboot', data))
        reboot_data = ['2920c8ec34f04f59b7df4284a4b41ca8cbec82ccdde331dd2d64cc89156af653', hash_dict['nonce']]
        self.factory.chain.state.db.put('reboot_data', reboot_data)
        blocknumber = hash_dict['blocknumber']
        logger.info(('Initiating Reboot Sequence..... #', blocknumber))
        if blocknumber != 0:
            if blocknumber <= self.factory.chain.height():
                self.factory.pos.update_node_state('unsynced')
                del self.factory.chain.m_blockchain[blocknumber:]
                self.factory.chain.f_write_m_blockchain()
                self.factory.chain.m_load_chain()
                self.factory.pos.update_node_state('synced')

    def MR(self, data):
        data = json.loads(data)
        if data['type'] not in MessageReceipt.allowed_types:
            return

        if data['type'] in ['R1', 'TX'] and self.factory.nodeState.state != 'synced':
            return

        if data['type'] == 'ST' and self.factory.chain.height() > 1 and self.factory.nodeState.state != 'synced':
            return

        if self.factory.master_mr.peer_contains_hash(data['hash'], data['type'], self):
            return

        self.factory.master_mr.add(data['hash'], data['type'], self)

        if data['hash'] in self.factory.master_mr.hash_callLater:  # Ignore if already requested
            return

        if self.factory.master_mr.contains(data['hash'], data['type']):
            return

        self.RFM(data)

    def RFM(self, data):  # Request full message, Move to factory
        msg_hash = data['hash']
        if msg_hash in self.factory.master_mr.hash_msg:
            if msg_hash in self.factory.master_mr.hash_callLater:
                del self.factory.master_mr.hash_callLater[msg_hash]
            return
        for peer in self.factory.master_mr.hash_peer[msg_hash]:
            if peer not in self.factory.master_mr.requested_hash[msg_hash]:
                self.factory.master_mr.requested_hash[msg_hash].append(peer)
                peer.transport.write(self.wrap_message('SFM', helper.json_encode(data)))
                call_later_obj = reactor.callLater(c.message_receipt_timeout,
                                                   self.RFM,
                                                   data)
                self.factory.master_mr.hash_callLater[msg_hash] = call_later_obj
                return

        # If executing reach to this line, then it means no peer was able to provide
        # Full message for this hash thus the hash has to be deleted.
        # Moreover, negative points could be added to the peers, for this behavior
        if msg_hash in self.factory.master_mr.hash_callLater:
            del self.factory.master_mr.hash_callLater[msg_hash]

    def SFM(self, data):  # Send full message
        data = json.loads(data)
        msg_hash = data['hash']
        msg_type = data['type']
        if not self.factory.master_mr.contains(msg_hash, msg_type):
            return

        # Sending message from node, doesn't guarantee that peer has received it.
        # Thus requesting peer could re request it, may be ACK would be required
        # To confirm, if the peer has received, otherwise X number of maximum retry
        # if self.factory.master_mr.peer_contains_hash(msg_hash, msg_type, self):
        #    return

        self.transport.write(self.wrap_message(msg_type,
                                               self.factory.master_mr.hash_msg[msg_hash]))<commit_msg>fixing missing references to config<commit_after># Distributed under the MIT software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.

# -features POS, quantum secure signature scheme..
from qrlcore import logger
from qrlcore.block import Block

__author__ = 'pete'
import json
import random
import struct
import time
from collections import Counter, defaultdict
from copy import deepcopy
from decimal import Decimal

from twisted.internet import reactor
from twisted.internet.protocol import ServerFactory, Protocol

import fork
import helper
from messagereceipt import MessageReceipt
from qrlcore.merkle import sha256
from qrlcore.transaction import StakeTransaction, SimpleTransaction
import configuration as config


class NodeState:
    def __init__(self):
        self.state = 'unsynced'
        self.epoch_diff = -1


class POS:
    def __init__(self, chain, p2pFactory, nodeState, ntp):
        self.master_mr = MessageReceipt()
        self.nodeState = nodeState
        self.ntp = ntp
        self.chain = chain
        self.r1_time_diff = defaultdict(list)
        self.r2_time_diff = defaultdict(list)

        self.incoming_blocks = {}
        self.last_pos_cycle = 0
        self.last_selected_height = 0
        self.last_bk_time = 0
        self.last_pb_time = 0
        self.next_header_hash = None
        self.next_block_number = None
        self.fmbh_allowed_peers = {}
        self.fmbh_blockhash_peers = {}

        self.p2pFactory = p2pFactory

    def update_node_state(self, state):
        self.nodeState.state = state
        logger.info(('Status changed to ', self.nodeState.state))
        if self.nodeState.state == 'synced':
            self.nodeState.epoch_diff = 0
            self.last_pos_cycle = time.time()
            self.restart_post_block_logic()
        elif self.nodeState.state == 'unsynced':
            self.last_bk_time = time.time()
            self.restart_unsynced_logic()
        elif self.nodeState.state == 'forked':
            self.stop_post_block_logic()
        elif self.nodeState.state == 'syncing':
            self.last_pb_time = time.time()

    def stop_monitor_bk(self):
        try:
            reactor.monitor_bk.cancel()
        except:
            pass

    def restart_monitor_bk(self, delay=60):
        self.stop_monitor_bk()
        reactor.monitor_bk = reactor.callLater(delay, self.monitor_bk)

    def monitor_bk(self):
        if (self.nodeState.state == 'synced' or self.nodeState.state == 'unsynced') and 90 < time.time() - self.last_pos_cycle:
            if self.nodeState.state == 'synced':
                self.stop_post_block_logic()
                self.reset_everything()
                self.update_node_state('unsynced')
                self.epoch_diff = -1
            elif time.time() - self.last_bk_time > 120:
                self.last_pos_cycle = time.time()
                logger.info(' POS cycle activated by monitor_bk() ')
                self.update_node_state('synced')

        if self.nodeState.state == 'syncing' and time.time() - self.last_pb_time > 60:
            self.stop_post_block_logic()
            self.reset_everything()
            self.update_node_state('unsynced')
            self.epoch_diff = -1
        reactor.monitor_bk = reactor.callLater(60, self.monitor_bk)

    def peers_blockheight_headerhash(self):
        for peer in self.p2pFactory.peers:
            peer.fetch_headerhash_n(self.chain.m_blockheight())

    def check_fork_status(self):
        current_height = self.chain.m_blockheight()
        block_hash_counter = Counter()
        for peer in self.p2pFactory.peers:
            if current_height in peer.blocknumber_headerhash.keys():
                block_hash_counter[peer.blocknumber_headerhash[current_height]] += 1

        blockhash = block_hash_counter.most_common(1)
        if blockhash:
            blockhash = blockhash[0][0]
            actual_blockhash = self.chain.m_get_block(current_height).blockheader.headerhash
            if actual_blockhash != blockhash:
                logger.info('Blockhash didnt matched in peers_blockheight()')
                logger.info(('Local blockhash - ', actual_blockhash))
                logger.info(('Consensus blockhash - ', blockhash))
                fork.fork_recovery(current_height, self.chain, self.randomize_headerhash_fetch)
                return True
        return

    def peers_blockheight(self):
        if self.nodeState.state == 'syncing':
            return
        if self.check_fork_status():
            return

        block_height_counter = Counter()

        for peer in self.p2pFactory.peers:
            block_height_counter[peer.blockheight] += 1

        blocknumber = block_height_counter.most_common(1)
        if not blocknumber:
            return  # TODO : Re-Schedule with delay

        blocknumber = blocknumber[0][0]

        if blocknumber > self.chain.height():  # chain.m_blockheight():  len(chain.m_blockchain)
            # pending_blocks['target'] = blocknumber
            logger.info(('Calling downloader from peers_blockheight due to no POS CYCLE ', blocknumber))
            logger.info(('Download block from ', self.chain.height() + 1, ' to ', blocknumber))
            self.last_pb_time = time.time()
            self.update_node_state('syncing')
            self.randomize_block_fetch(self.chain.height() + 1)
        return

    def schedule_peers_blockheight(self, delay=100):
        try:
            reactor.peers_blockheight.cancel()
        except Exception:
            pass
        reactor.peers_blockheight = reactor.callLater(delay, self.peers_blockheight)
        try:
            reactor.peers_blockheight_headerhash.cancel()
        except Exception:
            pass
        reactor.peers_blockheight_headerhash = reactor.callLater(70, self.peers_blockheight_headerhash)

    # pos functions. an asynchronous loop.

    # first block 1 is created with the stake list for epoch 0 decided from circulated st transactions

    def pre_pos_1(self, data=None):  # triggered after genesis for block 1..
        logger.info(('pre_pos_1'))
        # are we a staker in the stake list?

        if self.chain.mining_address in self.chain.m_blockchain[0].stake_list:
            logger.info(('mining address:', self.chain.mining_address, ' in the genesis.stake_list'))

            self.chain.my[0][1].hashchain(epoch=0)
            self.chain.hash_chain = self.chain.my[0][1].hc
            self.chain.block_chain_buffer.hash_chain[0] = self.chain.my[0][1].hc

            logger.info(('hashchain terminator: ', self.chain.my[0][1].hc_terminator))
            st = StakeTransaction().create_stake_transaction(mining_address=self.chain.mining_address,
                                                             blocknumber=0,
                                                             data=self.chain.my[0][1],
                                                             hashchain_terminator=self.chain.my[0][1].hc_terminator,
                                                             first_hash=self.chain.my[0][1].hc[-1][-2],
                                                             balance=self.chain.state.state_balance(self.chain.mining_address))
            self.chain.wallet.f_save_winfo()
            self.chain.add_st_to_pool(st)
            # send the stake tx to generate hashchain terminators for the staker addresses..
            self.p2pFactory.send_st_to_peers(st)
            logger.info('await delayed call to build staker list from genesis')
            reactor.callLater(5, self.pre_pos_2, st)
            return

        logger.info('not in stake list..no further pre_pos_x calls')
        return

    def pre_pos_2(self, data=None):
        logger.info(('pre_pos_2'))
        if self.chain.height() >= 1:
            return
        # assign hash terminators to addresses and generate a temporary stake list ordered by st.hash..

        tmp_list = []

        for st in self.chain.stake_pool:
            if st.txfrom in self.chain.m_blockchain[0].stake_list:
                tmp_list.append([st.txfrom, st.hash, 0, st.first_hash, self.chain.state.state_balance(st.txfrom)])

        # required as doing chain.stake_list.index(s) which will result into different number on different server
        self.chain.block_chain_buffer.epoch_seed = self.chain.state.calc_seed(tmp_list)
        self.chain.stake_list = sorted(tmp_list,
                                       key=lambda staker: self.chain.score(stake_address=staker[0],
                                                                           reveal_one=sha256(str(staker[1])),
                                                                           balance=self.chain.state.state_balance(st.txfrom),
                                                                           seed=self.chain.block_chain_buffer.epoch_seed))

        logger.info(('genesis stakers ready = ', len(self.chain.stake_list), '/', config.dev.minimum_required_stakers))
        logger.info(('node address:', self.chain.mining_address))

        if len(self.chain.stake_list) < config.dev.minimum_required_stakers:  # stake pool still not full..reloop..
            self.p2pFactory.send_st_to_peers(data)
            logger.info('waiting for stakers.. retry in 5s')
            reactor.callID = reactor.callLater(5, self.pre_pos_2, data)
            return
        logger.info((str(self.chain.stake_list)))

        if self.chain.mining_address == self.chain.stake_list[0][0]:
            logger.info('designated to create block 1: building block..')

            # create the genesis block 2 here..
            my_hash_chain, _ = self.chain.select_hashchain(self.chain.m_blockchain[-1].blockheader.headerhash,
                                                           self.chain.mining_address, self.chain.my[0][1].hc,
                                                           blocknumber=1)
            b = self.chain.m_create_block(my_hash_chain[-2])
            self.pre_block_logic(b)
        else:
            logger.info(('await block creation by stake validator:', self.chain.stake_list[0][0]))
            self.last_bk_time = time.time()
            self.restart_unsynced_logic()
        return

    def process_transactions(self, num):
        tmp_num = num
        for tx in self.chain.pending_tx_pool:
            tmp_num -= 1
            tx_peer = tx[1]
            tx = tx[0]
            if tx.validate_tx() != True:
                logger.info(('>>>TX ', tx.txhash, 'failed validate_tx'))
                continue

            isValidState = tx.state_validate_tx(
                state=self.chain.state,
                transaction_pool=self.chain.transaction_pool
            )
            if not isValidState:
                logger.info(('>>>TX', tx.txhash, 'failed state_validate'))
                continue

            logger.info(('>>>TX - ', tx.txhash, ' from - ', tx_peer.transport.getPeer().host, ' relaying..'))
            self.chain.add_tx_to_pool(tx)

            txn_msg = tx_peer.wrap_message('TX', tx.transaction_to_json())
            for peer in tx_peer.factory.peers:
                if peer != tx_peer:
                    peer.transport.write(txn_msg)

        for i in range(num - tmp_num):
            del self.chain.pending_tx_pool[0]
            del self.chain.pending_tx_pool_hash[0]

    # create new block..

    def create_new_block(self, winner, reveals, vote_hashes, last_block_number):
        logger.info(('create_new_block #', (last_block_number + 1)))
        tx_list = []
        for t in self.chain.transaction_pool:
            tx_list.append(t.txhash)
        block_obj = self.chain.create_stake_block(tx_list, winner, reveals, vote_hashes, last_block_number)

        return block_obj

    def reset_everything(self, data=None):
        logger.info('** resetting loops and emptying chain.stake_reveal_one and chain.expected_winner ')
        for r in self.chain.stake_reveal_one:
            msg_hash = r[5]
            self.master_mr.deregister(msg_hash, 'R1')

        del self.chain.stake_reveal_one[:]
        return

    def filter_reveal_one_two(self, blocknumber=None):
        if not blocknumber:
            blocknumber = self.chain.m_blockchain[-1].blockheader.blocknumber

        self.chain.stake_reveal_one = filter(lambda s: s[2] > blocknumber,
                                             self.chain.stake_reveal_one)

        return

    # TODO: Incomplete fn, use to select the maximum blockheight by consensus
    def select_blockheight_by_consensus(self):
        block_height_counter = Counter()
        # for identity in self.fmbh_allowed_peers:
        #    block_height_counter[s[2]] += 1
        target_block_height = block_height_counter.most_common(1)

        if len(target_block_height) == 0:
            return None

        last_selected_height = target_block_height[0][0]
        return last_selected_height

    '''
    Unsynced Logic
    1.	Request for maximum blockheight and passes bock number X
    2.	Peers response chain height with headerhash and the headerhash of block number X
    3.	Unsynced node, selects most common chain height, matches the headerhash of block number X
    4.	If headerhash of block number X doesn't match, change state to Forked
    5.	If headerhash of block number X matches, perform Downloading of blocks from those selected peers
    '''

    def restart_unsynced_logic(self, delay=0):
        try:
            reactor.unsynced_logic.cancel()
        except:
            pass
        reactor.unsynced_logic = reactor.callLater(delay, self.unsynced_logic)

    def unsynced_logic(self):
        if self.nodeState.state == 'synced':
            return

        self.fmbh_blockhash_peers = {}
        self.fmbh_allowed_peers = {}
        for peer in self.p2pFactory.peers:
            self.fmbh_allowed_peers[peer.identity] = None
            peer.fetch_FMBH()
        reactor.unsynced_logic = reactor.callLater(20, self.start_download)

    def start_download(self):
        # add peers and their identity to requested list
        # FMBH
        if self.nodeState.state == 'synced':
            return
        logger.info(('Checking Download..'))
        '''
        global fmbh_blockhash_peers
        max_height = None
        selected_blockhash = None
        for blockheaderhash in fmbh_blockhash_peers:
            if fmbh_blockhash_peers[blockheaderhash]['blocknumber']>max_height:
                max_height = fmbh_blockhash_peers[blockheaderhash]['blocknumber']
                selected_blockhash = blockheaderhash
        for peer in fmbh_blockhash_peers[selected_blockhash]['peers']:
            f.target_peers = {}
            f.target_peers[peer.identity] = peer
        
        if max_height == None or max_height<=chain.height():
            chain.state.update('synced')
            return
        
        chain.state.update('syncing')
        pending_blocks['start_block'] = chain.m_blockchain[-1].blockheader.blocknumber
        pending_blocks['target'] = fmbh_blockhash_peers[selected_blockhash]['blocknumber']
        pending_blocks['headerhash'] = selected_blockhash
        randomize_block_fetch(chain.height() + 1)
        '''
        max = -1
        max_headerhash = None
        for headerhash in self.fmbh_blockhash_peers:
            if self.fmbh_blockhash_peers[headerhash]['blocknumber'] > self.chain.height():
                if len(self.fmbh_blockhash_peers[headerhash]['peers']) > max:
                    max = len(self.fmbh_blockhash_peers[headerhash]['peers'])
                    max_headerhash = headerhash

        # Adding all peers
        # TODO only trusted peer
        # for peer in self.p2pFactory.peers:
        if not max_headerhash:
            logger.info('No peers responded FMBH request')
            return
        for peer in self.fmbh_blockhash_peers[max_headerhash]['peers']:
            self.p2pFactory.target_peers[peer.identity] = peer
        self.update_node_state('syncing')
        self.randomize_block_fetch(self.chain.height() + 1)

    def pre_block_logic(self, block, peer_identity=None):
        if len(self.chain.m_blockchain) == 0:
            self.chain.m_read_chain()

        blocknumber = block.blockheader.blocknumber
        headerhash = block.blockheader.headerhash
        prev_blockheaderhash = block.blockheader.prev_blockheaderhash
        curr_epoch = self.chain.height() /config.dev.blocks_per_epoch
        next_epoch = (self.chain.height() + 1) /config.dev.blocks_per_epoch
        chain_buffer_height = self.chain.block_chain_buffer.height()

        if blocknumber <= self.chain.height():
            return False

        if self.nodeState.state == 'synced':
            if self.chain.block_chain_buffer.add_block(block):
                self.p2pFactory.send_block_to_peers(block, peer_identity)
        else:
            if chain_buffer_height + 1 == blocknumber:
                if blocknumber > 1 and self.chain.block_chain_buffer.add_block(block):
                    self.p2pFactory.send_block_to_peers(block, peer_identity)
                elif blocknumber == 1 and self.chain.block_chain_buffer.add_block_mainchain(block):
                    self.p2pFactory.send_block_to_peers(block, peer_identity)
                self.update_node_state('synced')
            else:
                self.chain.block_chain_buffer.add_pending_block(block)

        if self.nodeState.state == 'synced':
            if chain_buffer_height + 1 == blocknumber:
                self.last_pos_cycle = time.time()
                block_timestamp = int(block.blockheader.timestamp)
                curr_time = int(self.ntp.getTime())
                delay = config.dev.POS_delay_after_block - min(config.dev.POS_delay_after_block, max(0, curr_time - block_timestamp))

                self.restart_post_block_logic(delay)
        # commented
        '''
        if chain.m_blockchain and block.blockheader.epoch == curr_epoch or block.blockheader.epoch == next_epoch:
            if blocknumber < chain.height() + 1 and headerhash != chain.m_blockchain[blocknumber].blockheader.headerhash:
                if not chain.validate_block(block):
                    return False

                sum_block_reward_old_sl, sum_block_reward_new_sl = sumBlockReward(chain.m_blockchain[block.blockheader.blocknumber].blockheader.stake_selector, block.blockheader.stake_selector, block.blockheader.blocknumber)

                #Compare if strongest, then replace from the buffer
                total_stakers_new_block, score_new_block = getBlockStakeInfo(block, block_reward=sum_block_reward_new_sl)
                total_stakers, score = getBlockStakeInfo(chain.m_blockchain[blocknumber], block_reward=sum_block_reward_old_sl)
                printL (( 'New Block # ', blocknumber, ' HeaderHash: ', headerhash, ' Score: ', score_new_block, ' sum block reward : ', sum_block_reward_new_sl ))
                printL (( 'Old Block # ', blocknumber, ' HeaderHash: ', chain.m_blockchain[blocknumber].blockheader.headerhash, ' Score: ', score, ' sum block reward : ', sum_block_reward_old_sl ))
                printL (( str(chain.state_balance(block.blockheader.stake_selector)) ))
                if score_new_block < score:  #or total_stakers_new_block>total_stakers#Need to be reviewed
                    tmp_blocks = chain.m_blockchain[blocknumber:][::-1]
                    for tmp_block in tmp_blocks:
                        update_nonce(tmp_block)
                        txn_block_to_pool(tmp_block)
                        del chain.m_blockchain[tmp_block.blockheader.blocknumber]
                    add_block(block)
                    if chain.state.current == 'synced':
                        f.send_block_to_peers(block)

                return True

            if blocknumber == chain.height()+1 and prev_blockheaderhash == chain.m_blockchain[-1].blockheader.headerhash:
                if not chain.validate_block(block):
                    return False

                add_block(block)

                if chain.state.current == 'synced':
                    f.send_block_to_peers(block)
                if chain.state.current == 'unsynced':
                    chain.state.update('synced')
                restart_post_block_logic()
                return True
        '''
        return True

    def stop_post_block_logic(self, delay=0):
        try:
            reactor.post_block_logic.cancel()
            reactor.prepare_winners.cancel()
        except Exception:
            pass

    def restart_post_block_logic(self, delay=0):
        self.stop_post_block_logic()
        reactor.post_block_logic = reactor.callLater(delay,
                                                     self.post_block_logic)

    # post block logic we initiate the next POS cycle, send R1, send ST, reset POS flags and remove unnecessary messages in chain.stake_reveal_one and _two..

    def post_block_logic(self):
        self.filter_reveal_one_two()

        our_reveal = None
        blocknumber = self.chain.block_chain_buffer.height() + 1

        if self.p2pFactory.stake:
            tmp_stake_list = [
                s[0] for s in self.chain.block_chain_buffer.stake_list_get(blocknumber)
            ]
            if self.chain.mining_address in tmp_stake_list:
                our_reveal = self.p2pFactory.send_stake_reveal_one(blocknumber)
                self.schedule_prepare_winners(our_reveal, blocknumber - 1, 30)

            '''tmp_next_stake_list = [
                s[0] for s in self.chain.block_chain_buffer.next_stake_list_get(blocknumber)
            ]'''
            next_stake_list = self.chain.block_chain_buffer.next_stake_list_get(blocknumber)
            next_stake_first_hash = {}
            for s in next_stake_list:
                next_stake_first_hash[s[0]] = s[3]

            epoch = blocknumber // config.dev.blocks_per_epoch
            epoch_blocknum = blocknumber - epoch * config.dev.blocks_per_epoch

            if epoch_blocknum < config.dev.stake_before_x_blocks and self.chain.mining_address not in next_stake_first_hash:
                diff = max(1, ((config.dev.stake_before_x_blocks - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
                if random.randint(1, diff) == 1:
                    self.make_st_tx(blocknumber, None)
            elif epoch_blocknum >= config.dev.stake_before_x_blocks - 1 and self.chain.mining_address in next_stake_first_hash:
                if next_stake_first_hash[self.chain.mining_address] is None:
                    threshold_blocknum = self.chain.state.get_staker_threshold_blocknum(next_stake_list,
                                                                                        self.chain.mining_address)
                    max_threshold_blocknum = config.dev.blocks_per_epoch
                    if threshold_blocknum == config.dev.low_staker_first_hash_block:
                        max_threshold_blocknum = config.dev.high_staker_first_hash_block

                    if epoch_blocknum >= threshold_blocknum - 1 and epoch_blocknum < max_threshold_blocknum - 1:
                        diff = max(1, ((max_threshold_blocknum - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
                        if random.randint(1, diff) == 1:
                            my = deepcopy(self.chain.my[0][1])
                            my.hashchain(epoch=epoch + 1)
                            self.make_st_tx(blocknumber, my.hc[-1][-2])

        return

    def make_st_tx(self, blocknumber, first_hash):
        balance = self.chain.state.state_balance(self.chain.mining_address)
        if balance < config.dev.minimum_staking_balance_required:
            logger.info('Staking not allowed due to insufficient balance')
            logger.info(('Balance ', balance))
            return

        st = StakeTransaction().create_stake_transaction(
            self.chain.mining_address, blocknumber,
            self.chain.my[0][1],
            first_hash=first_hash,
            balance=balance
        )
        self.p2pFactory.send_st_to_peers(st)
        self.chain.wallet.f_save_winfo()
        for num in range(len(self.chain.stake_pool)):
            t = self.chain.stake_pool[num]
            if st.hash == t.hash:
                if st.get_message_hash() == t.get_message_hash():
                    return
                del self.chain.stake_pool[num]
                break

        self.chain.stake_pool.append(st)
        '''
        for t in self.chain.stake_pool:
            if st.hash == t.hash:
                if st.get_message_hash() != t.get_message_hash():
                    t.first_hash = st.first_hash
                return
        self.chain.add_st_to_pool(st)
        '''

    def schedule_prepare_winners(self, our_reveal, last_block_number, delay=0):
        try:
            reactor.prepare_winners.cancel()
        except:
            pass
        reactor.prepare_winners = reactor.callLater(
            delay,
            self.prepare_winners,
            our_reveal=our_reveal,
            last_block_number=last_block_number)

    def prepare_winners(self, our_reveal, last_block_number):
        if not self.nodeState.state == 'synced':
            return
        filtered_reveal_one = []
        reveals = []
        vote_hashes = []
        next_block_num = last_block_number + 1
        for s in self.chain.stake_reveal_one:
            tmp_strongest_headerhash = self.chain.block_chain_buffer.get_strongest_headerhash(last_block_number)
            if s[1] == tmp_strongest_headerhash and s[2] == next_block_num:
                filtered_reveal_one.append(s)
                reveals.append(s[3])
                vote_hashes.append(s[5])

        self.restart_post_block_logic(30)

        if len(filtered_reveal_one) <= 1:
            logger.info(('only received one reveal for this block.. blocknum #', next_block_num))
            return

        epoch = (next_block_num) / config.dev.blocks_per_epoch  # +1 = next block
        seed = self.chain.block_chain_buffer.get_epoch_seed(next_block_num)
        winners = self.chain.select_winners(filtered_reveal_one,
                                            topN=3,
                                            seed=seed)

        # reactor.process_blocks = reactor.callLater(30, process_blocks, winners=winners, our_reveal=our_reveal)

        if not (self.p2pFactory.stake and our_reveal):
            return

        if our_reveal in winners:
            block = self.create_new_block(our_reveal,
                                          reveals,
                                          vote_hashes,
                                          last_block_number)
            self.pre_block_logic(block)  # broadcast this block

        if self.chain.pending_tx_pool:
            if len(self.chain.transaction_pool) < 10:
                logger.info('Processing TXNs if any')
                self.process_transactions(5)

    def randomize_block_fetch(self, blocknumber):
        if self.nodeState.state != 'syncing' or blocknumber <= self.chain.height():
            return

        if len(self.p2pFactory.target_peers.keys()) == 0:
            logger.info(' No target peers found.. stopping download')
            return

        reactor.download_monitor = reactor.callLater(20,
                                                     self.randomize_block_fetch, blocknumber)

        random_peer = self.p2pFactory.target_peers[random.choice(self.p2pFactory.target_peers.keys())]
        random_peer.fetch_block_n(blocknumber)

    def randomize_headerhash_fetch(self, block_number):
        if self.nodeState.state != 'forked':
            return
        if block_number not in fork.pending_blocks or fork.pending_blocks[block_number][1] <= 10:  # retry only 11 times
            headerhash_monitor = reactor.callLater(15, self.randomize_headerhash_fetch, block_number)
            if len(self.p2pFactory.peers) > 0:
                try:
                    if len(self.p2pFactory.fork_target_peers) == 0:
                        for peer in self.p2pFactory.peers:
                            self.p2pFactory.fork_target_peers[peer.identity] = peer
                    if len(self.p2pFactory.fork_target_peers) > 0:
                        random_peer = self.p2pFactory.fork_target_peers[
                            random.choice(
                                self.p2pFactory.fork_target_peers.keys()
                            )
                        ]
                        count = 0
                        if block_number in fork.pending_blocks:
                            count = fork.pending_blocks[block_number][1] + 1
                        fork.pending_blocks[block_number] = [
                            random_peer.identity, count, None, headerhash_monitor
                        ]
                        random_peer.fetch_headerhash_n(block_number)
                except:
                    logger.info('Exception at randomize_headerhash_fetch')
            else:
                logger.info(('No peers connected.. Will try again... randomize_headerhash_fetch: ', block_number))
        else:
            self.update_node_state('unsynced')

    # blockheight map for connected nodes - when the blockheight seems up to date after a sync or error, we check all connected nodes to ensure all on same chain/height..
    # note - may not return correctly during a block propagation..
    # once working alter to identify fork better..

    def blockheight_map(self):

        # i = [block_number, headerhash, self.transport.getPeer().host]

        logger.info(('blockheight_map:'))
        logger.info((self.chain.blockheight_map))

        # first strip out any laggards..
        self.chain.blockheight_map = filter(
            lambda s: s[0] >= self.chain.m_blockheight(),
            self.chain.blockheight_map
        )

        result = True

        # next identify any node entries which are not exactly correct..

        for s in self.chain.blockheight_map:
            if s[0] == self.chain.m_blockheight():
                if s[1] == self.chain.m_blockchain[-1].blockheader.headerhash:
                    logger.info(('node: ', s[2], '@', s[0], 'w/:', s[1], 'OK'))
            elif s[0] > self.chain.m_blockheight():
                logger.info(('warning..', s[2], 'at blockheight', s[0]))
                result = False

        # wipe it..

        del self.chain.blockheight_map[:]

        return result


class P2PProtocol(Protocol):
    def __init__(self):
        self.service = {'reboot': self.reboot,
                        'MR': self.MR,
                        # 'RFM': self.RFM, only for internal usage
                        'SFM': self.SFM,
                        'TX': self.TX,
                        'ST': self.ST,
                        'BM': self.BM,
                        'BK': self.BK,
                        'PBB': self.PBB,
                        'PB': self.PB,
                        'PH': self.PH,
                        'LB': self.LB,
                        'FMBH': self.FMBH,
                        'PMBH': self.PMBH,
                        'MB': self.MB,
                        'CB': self.CB,
                        'BN': self.BN,
                        'FB': self.FB,
                        'FH': self.FH,
                        'PO': self.PO,
                        'PI': self.PI,
                        'PL': self.PL,
                        'RT': self.RT,
                        'PE': self.PE,
                        'VE': self.VE,
                        'R1': self.R1,
                        'IP': self.IP,
                        }
        self.buffer = ''
        self.messages = []
        self.identity = None
        self.blockheight = None
        self.version = ''
        self.blocknumber_headerhash = {}
        self.last_requested_blocknum = None
        self.fetch_tried = 0
        pass

    def parse_msg(self, data):
        try:
            jdata = json.loads(data)
        except:
            return

        func = jdata['type']

        if func not in self.service:
            return

        func = self.service[func]
        try:
            if 'data' in jdata:
                func(jdata['data'])
            else:
                func()
        except:
            logger.info("parse_msg Exception while calling ")
            logger.info(("Func name ", func))
            # logger.info(( "JSON data ", jdata ))
            pass

    def reboot(self, data):
        hash_dict = json.loads(data)
        if not ('hash' in hash_dict and 'nonce' in hash_dict and 'blocknumber' in hash_dict):
            return
        status, error = self.factory.chain.validate_reboot(hash_dict['hash'], hash_dict['nonce'])
        if not status:
            logger.info(('status ', status))
            logger.info(('error ', error))
            return
        for peer in self.factory.peers:
            if peer != self:
                peer.transport.write(self.wrap_message('reboot', data))
        reboot_data = ['2920c8ec34f04f59b7df4284a4b41ca8cbec82ccdde331dd2d64cc89156af653', hash_dict['nonce']]
        self.factory.chain.state.db.put('reboot_data', reboot_data)
        blocknumber = hash_dict['blocknumber']
        logger.info(('Initiating Reboot Sequence..... #', blocknumber))
        if blocknumber != 0:
            if blocknumber <= self.factory.chain.height():
                self.factory.pos.update_node_state('unsynced')
                del self.factory.chain.m_blockchain[blocknumber:]
                self.factory.chain.f_write_m_blockchain()
                self.factory.chain.m_load_chain()
                self.factory.pos.update_node_state('synced')

    def MR(self, data):
        data = json.loads(data)
        if data['type'] not in MessageReceipt.allowed_types:
            return

        if data['type'] in ['R1', 'TX'] and self.factory.nodeState.state != 'synced':
            return

        if data['type'] == 'ST' and self.factory.chain.height() > 1 and self.factory.nodeState.state != 'synced':
            return

        if self.factory.master_mr.peer_contains_hash(data['hash'], data['type'], self):
            return

        self.factory.master_mr.add(data['hash'], data['type'], self)

        if data['hash'] in self.factory.master_mr.hash_callLater:  # Ignore if already requested
            return

        if self.factory.master_mr.contains(data['hash'], data['type']):
            return

        self.RFM(data)

    def RFM(self, data):  # Request full message, Move to factory
        msg_hash = data['hash']
        if msg_hash in self.factory.master_mr.hash_msg:
            if msg_hash in self.factory.master_mr.hash_callLater:
                del self.factory.master_mr.hash_callLater[msg_hash]
            return
        for peer in self.factory.master_mr.hash_peer[msg_hash]:
            if peer not in self.factory.master_mr.requested_hash[msg_hash]:
                self.factory.master_mr.requested_hash[msg_hash].append(peer)
                peer.transport.write(self.wrap_message('SFM', helper.json_encode(data)))
                call_later_obj = reactor.callLater(config.dev.message_receipt_timeout,
                                                   self.RFM,
                                                   data)
                self.factory.master_mr.hash_callLater[msg_hash] = call_later_obj
                return

        # If executing reach to this line, then it means no peer was able to provide
        # Full message for this hash thus the hash has to be deleted.
        # Moreover, negative points could be added to the peers, for this behavior
        if msg_hash in self.factory.master_mr.hash_callLater:
            del self.factory.master_mr.hash_callLater[msg_hash]

    def SFM(self, data):  # Send full message
        data = json.loads(data)
        msg_hash = data['hash']
        msg_type = data['type']
        if not self.factory.master_mr.contains(msg_hash, msg_type):
            return

        # Sending message from node, doesn't guarantee that peer has received it.
        # Thus requesting peer could re request it, may be ACK would be required
        # To confirm, if the peer has received, otherwise X number of maximum retry
        # if self.factory.master_mr.peer_contains_hash(msg_hash, msg_type, self):
        #    return

        self.transport.write(self.wrap_message(msg_type,
                                               self.factory.master_mr.hash_msg[msg_hash]))
{'fraction_non_alphanumeric': 0.09164785553047404, 'fraction_numerical': 0.007469731171762774, 'mean_word_length': 1.9282910023235318, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 116, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7081170', 'n_tokens_mistral': 21398, 'n_tokens_neox': 20351, 'n_words': 4067}