<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test behavior of headers messages to announce blocks.

Setup:

- Two nodes, two p2p connections to node0. One p2p connection should only ever
  receive inv's (omitted from testing description below, this is our control).
  Second node is used for creating reorgs.

test_null_locators
==================

Sends two getheaders requests with null locator values. First request's hashstop
value refers to validated block, while second request's hashstop value refers to
a block which hasn't been validated. Verifies only the first request returns
headers.

test_nonnull_locators
=====================

Part 1: No headers announcements before "sendheaders"
a. node mines a block [expect: inv]
   send getdata for the block [expect: block]
b. node mines another block [expect: inv]
   send getheaders and getdata [expect: headers, then block]
c. node mines another block [expect: inv]
   peer mines a block, announces with header [expect: getdata]
d. node mines another block [expect: inv]

Part 2: After "sendheaders", headers announcements should generally work.
a. peer sends sendheaders [expect: no response]
   peer sends getheaders with current tip [expect: no response]
b. node mines a block [expect: tip header]
c. for N in 1, ..., 10:
   * for announce-type in {inv, header}
     - peer mines N blocks, announces with announce-type
       [ expect: getheaders/getdata or getdata, deliver block(s) ]
     - node mines a block [ expect: 1 header ]

Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
- For response-type in {inv, getheaders}
  * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
  * node mines an 8-block reorg [ expect: inv at tip ]
  * peer responds with getblocks/getdata [expect: inv, blocks ]
  * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
  * node mines another block at tip [ expect: inv ]
  * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
  * peer requests block [ expect: block ]
  * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
  * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
  * node mines 1 block [expect: 1 header, peer responds with getdata]

Part 4: Test direct fetch behavior
a. Announce 2 old block headers.
   Expect: no getdata requests.
b. Announce 3 new blocks via 1 headers message.
   Expect: one getdata request for all 3 blocks.
   (Send blocks.)
c. Announce 1 header that forks off the last two blocks.
   Expect: no response.
d. Announce 1 more header that builds on that fork.
   Expect: one getdata request for two blocks.
e. Announce 16 more headers that build on that fork.
   Expect: getdata request for 14 more blocks.
f. Announce 1 more header that builds on that fork.
   Expect: no response.

Part 5: Test handling of headers that don't connect.
a. Repeat 10 times:
   1. Announce a header that doesn't connect.
      Expect: getheaders message
   2. Send headers chain.
      Expect: getdata for the missing blocks, tip update.
b. Then send 9 more headers that don't connect.
   Expect: getheaders message each time.
c. Announce a header that does connect.
   Expect: no response.
d. Announce 49 headers that don't connect.
   Expect: getheaders message each time.
e. Announce one more that doesn't connect.
   Expect: disconnect.
"""
from test_framework.blocktools import create_block, create_coinbase
from test_framework.mininode import (
    CBlockHeader,
    CInv,
    NODE_WITNESS,
    NetworkThread,
    NodeConnCB,
    mininode_lock,
    msg_block,
    msg_getblocks,
    msg_getdata,
    msg_getheaders,
    msg_headers,
    msg_inv,
    msg_sendheaders,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    sync_blocks,
    wait_until,
)

DIRECT_FETCH_RESPONSE_TIME = 0.05

class BaseNode(NodeConnCB):
    def __init__(self):
        super().__init__()
        self.block_announced = False
        self.last_blockhash_announced = None

    def send_get_data(self, block_hashes):
        """Request data for a list of block hashes."""
        msg = msg_getdata()
        for x in block_hashes:
            msg.inv.append(CInv(2, x))
        self.connection.send_message(msg)

    def send_get_headers(self, locator, hashstop):
        msg = msg_getheaders()
        msg.locator.vHave = locator
        msg.hashstop = hashstop
        self.connection.send_message(msg)

    def send_block_inv(self, blockhash):
        msg = msg_inv()
        msg.inv = [CInv(2, blockhash)]
        self.connection.send_message(msg)

    def send_header_for_blocks(self, new_blocks):
        headers_message = msg_headers()
        headers_message.headers = [CBlockHeader(b) for b in new_blocks]
        self.send_message(headers_message)

    def send_getblocks(self, locator):
        getblocks_message = msg_getblocks()
        getblocks_message.locator.vHave = locator
        self.send_message(getblocks_message)

    def wait_for_getdata(self, hash_list, timeout=60):
        if hash_list != []:
            test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list
            wait_until(test_function, timeout=timeout, lock=mininode_lock)

    def wait_for_block_announcement(self, block_hash, timeout=60):
        test_function = lambda: self.last_blockhash_announced == block_hash
        wait_until(test_function, timeout=timeout, lock=mininode_lock)

    def on_inv(self, conn, message):
        self.block_announced = True
        self.last_blockhash_announced = message.inv[-1].hash

    def on_headers(self, conn, message):
        if len(message.headers):
            self.block_announced = True
            message.headers[-1].calc_sha256()
            self.last_blockhash_announced = message.headers[-1].sha256

    def clear_last_announcement(self):
        with mininode_lock:
            self.block_announced = False
            self.last_message.pop("inv", None)
            self.last_message.pop("headers", None)

    def check_last_announcement(self, headers=None, inv=None):
        """Test whether the last announcement received had the right header or the right inv.

        inv and headers should be lists of block hashes."""

        test_function = lambda: self.block_announced
        wait_until(test_function, timeout=60, lock=mininode_lock)

        with mininode_lock:
            self.block_announced = False

            compare_inv = []
            if "inv" in self.last_message:
                compare_inv = [x.hash for x in self.last_message["inv"].inv]
            if inv is not None:
                assert_equal(compare_inv, inv)

            compare_headers = []
            if "headers" in self.last_message:
                compare_headers = [x.sha256 for x in self.last_message["headers"].headers]
            if headers is not None:
                assert_equal(compare_headers, headers)

            self.last_message.pop("inv", None)
            self.last_message.pop("headers", None)

class SendHeadersTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 2

    def mine_blocks(self, count):
        """Mine count blocks and return the new tip."""

        # Clear out last block announcement from each p2p listener
        [x.clear_last_announcement() for x in self.nodes[0].p2ps]
        self.nodes[0].generate(count)
        return int(self.nodes[0].getbestblockhash(), 16)

    def mine_reorg(self, length):
        """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).

        Note: we clear the state of our p2p connections after the
        to-be-reorged-out blocks are mined, so that we don't break later tests.
        return the list of block hashes newly mined."""

        self.nodes[0].generate(length)  # make sure all invalidated blocks are node0's
        sync_blocks(self.nodes, wait=0.1)
        for x in self.nodes[0].p2ps:
            x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
            x.clear_last_announcement()

        tip_height = self.nodes[1].getblockcount()
        hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
        self.nodes[1].invalidateblock(hash_to_invalidate)
        all_hashes = self.nodes[1].generate(length + 1)  # Must be longer than the orig chain
        sync_blocks(self.nodes, wait=0.1)
        return [int(x, 16) for x in all_hashes]

    def run_test(self):
        # Setup the p2p connections and start up the network thread.
        inv_node = self.nodes[0].add_p2p_connection(BaseNode())
        # Set nServices to 0 for test_node, so no block download will occur outside of
        # direct fetching
        test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)

        NetworkThread().start()  # Start up network handling in another thread

        # Test logic begins here
        inv_node.wait_for_verack()
        test_node.wait_for_verack()

        # Ensure verack's have been processed by our peer
        inv_node.sync_with_ping()
        test_node.sync_with_ping()

        self.test_null_locators(test_node)
        self.test_nonnull_locators(test_node, inv_node)

    def test_null_locators(self, test_node):
        tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
        tip_hash = int(tip["hash"], 16)

        self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
        test_node.clear_last_announcement()
        test_node.send_get_headers(locator=[], hashstop=tip_hash)
        test_node.check_last_announcement(headers=[tip_hash])

        self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
        block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
        block.solve()
        test_node.send_header_for_blocks([block])
        test_node.clear_last_announcement()
        test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
        test_node.sync_with_ping()
        assert_equal(test_node.block_announced, False)
        test_node.send_message(msg_block(block))

    def test_nonnull_locators(self, test_node, inv_node):
        tip = int(self.nodes[0].getbestblockhash(), 16)

        # PART 1
        # 1. Mine a block; expect inv announcements each time
        self.log.info("Part 1: headers don't start before sendheaders message...")
        for i in range(4):
            old_tip = tip
            tip = self.mine_blocks(1)
            inv_node.check_last_announcement(inv=[tip], headers=[])
            test_node.check_last_announcement(inv=[tip], headers=[])
            # Try a few different responses; none should affect next announcement
            if i == 0:
                # first request the block
                test_node.send_get_data([tip])
                test_node.wait_for_block(tip)
            elif i == 1:<commit_msg>[tests] fix flakiness in sendheaders.py<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test behavior of headers messages to announce blocks.

Setup:

- Two nodes, two p2p connections to node0. One p2p connection should only ever
  receive inv's (omitted from testing description below, this is our control).
  Second node is used for creating reorgs.

test_null_locators
==================

Sends two getheaders requests with null locator values. First request's hashstop
value refers to validated block, while second request's hashstop value refers to
a block which hasn't been validated. Verifies only the first request returns
headers.

test_nonnull_locators
=====================

Part 1: No headers announcements before "sendheaders"
a. node mines a block [expect: inv]
   send getdata for the block [expect: block]
b. node mines another block [expect: inv]
   send getheaders and getdata [expect: headers, then block]
c. node mines another block [expect: inv]
   peer mines a block, announces with header [expect: getdata]
d. node mines another block [expect: inv]

Part 2: After "sendheaders", headers announcements should generally work.
a. peer sends sendheaders [expect: no response]
   peer sends getheaders with current tip [expect: no response]
b. node mines a block [expect: tip header]
c. for N in 1, ..., 10:
   * for announce-type in {inv, header}
     - peer mines N blocks, announces with announce-type
       [ expect: getheaders/getdata or getdata, deliver block(s) ]
     - node mines a block [ expect: 1 header ]

Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
- For response-type in {inv, getheaders}
  * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
  * node mines an 8-block reorg [ expect: inv at tip ]
  * peer responds with getblocks/getdata [expect: inv, blocks ]
  * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
  * node mines another block at tip [ expect: inv ]
  * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
  * peer requests block [ expect: block ]
  * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
  * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
  * node mines 1 block [expect: 1 header, peer responds with getdata]

Part 4: Test direct fetch behavior
a. Announce 2 old block headers.
   Expect: no getdata requests.
b. Announce 3 new blocks via 1 headers message.
   Expect: one getdata request for all 3 blocks.
   (Send blocks.)
c. Announce 1 header that forks off the last two blocks.
   Expect: no response.
d. Announce 1 more header that builds on that fork.
   Expect: one getdata request for two blocks.
e. Announce 16 more headers that build on that fork.
   Expect: getdata request for 14 more blocks.
f. Announce 1 more header that builds on that fork.
   Expect: no response.

Part 5: Test handling of headers that don't connect.
a. Repeat 10 times:
   1. Announce a header that doesn't connect.
      Expect: getheaders message
   2. Send headers chain.
      Expect: getdata for the missing blocks, tip update.
b. Then send 9 more headers that don't connect.
   Expect: getheaders message each time.
c. Announce a header that does connect.
   Expect: no response.
d. Announce 49 headers that don't connect.
   Expect: getheaders message each time.
e. Announce one more that doesn't connect.
   Expect: disconnect.
"""
from test_framework.blocktools import create_block, create_coinbase
from test_framework.mininode import (
    CBlockHeader,
    CInv,
    NODE_WITNESS,
    NetworkThread,
    NodeConnCB,
    mininode_lock,
    msg_block,
    msg_getblocks,
    msg_getdata,
    msg_getheaders,
    msg_headers,
    msg_inv,
    msg_sendheaders,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    sync_blocks,
    wait_until,
)

DIRECT_FETCH_RESPONSE_TIME = 0.05

class BaseNode(NodeConnCB):
    def __init__(self):
        super().__init__()
        self.block_announced = False
        self.last_blockhash_announced = None

    def send_get_data(self, block_hashes):
        """Request data for a list of block hashes."""
        msg = msg_getdata()
        for x in block_hashes:
            msg.inv.append(CInv(2, x))
        self.connection.send_message(msg)

    def send_get_headers(self, locator, hashstop):
        msg = msg_getheaders()
        msg.locator.vHave = locator
        msg.hashstop = hashstop
        self.connection.send_message(msg)

    def send_block_inv(self, blockhash):
        msg = msg_inv()
        msg.inv = [CInv(2, blockhash)]
        self.connection.send_message(msg)

    def send_header_for_blocks(self, new_blocks):
        headers_message = msg_headers()
        headers_message.headers = [CBlockHeader(b) for b in new_blocks]
        self.send_message(headers_message)

    def send_getblocks(self, locator):
        getblocks_message = msg_getblocks()
        getblocks_message.locator.vHave = locator
        self.send_message(getblocks_message)

    def wait_for_getdata(self, hash_list, timeout=60):
        if hash_list != []:
            test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list
            wait_until(test_function, timeout=timeout, lock=mininode_lock)

    def wait_for_block_announcement(self, block_hash, timeout=60):
        test_function = lambda: self.last_blockhash_announced == block_hash
        wait_until(test_function, timeout=timeout, lock=mininode_lock)

    def on_inv(self, conn, message):
        self.block_announced = True
        self.last_blockhash_announced = message.inv[-1].hash

    def on_headers(self, conn, message):
        if len(message.headers):
            self.block_announced = True
            message.headers[-1].calc_sha256()
            self.last_blockhash_announced = message.headers[-1].sha256

    def clear_last_announcement(self):
        with mininode_lock:
            self.block_announced = False
            self.last_message.pop("inv", None)
            self.last_message.pop("headers", None)

    def check_last_announcement(self, headers=None, inv=None):
        """Test whether the last announcement received had the right header or the right inv.

        inv and headers should be lists of block hashes."""

        test_function = lambda: self.block_announced
        wait_until(test_function, timeout=60, lock=mininode_lock)

        with mininode_lock:
            self.block_announced = False

            compare_inv = []
            if "inv" in self.last_message:
                compare_inv = [x.hash for x in self.last_message["inv"].inv]
            if inv is not None:
                assert_equal(compare_inv, inv)

            compare_headers = []
            if "headers" in self.last_message:
                compare_headers = [x.sha256 for x in self.last_message["headers"].headers]
            if headers is not None:
                assert_equal(compare_headers, headers)

            self.last_message.pop("inv", None)
            self.last_message.pop("headers", None)

class SendHeadersTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 2

    def mine_blocks(self, count):
        """Mine count blocks and return the new tip."""

        # Clear out last block announcement from each p2p listener
        [x.clear_last_announcement() for x in self.nodes[0].p2ps]
        self.nodes[0].generate(count)
        return int(self.nodes[0].getbestblockhash(), 16)

    def mine_reorg(self, length):
        """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).

        Note: we clear the state of our p2p connections after the
        to-be-reorged-out blocks are mined, so that we don't break later tests.
        return the list of block hashes newly mined."""

        self.nodes[0].generate(length)  # make sure all invalidated blocks are node0's
        sync_blocks(self.nodes, wait=0.1)
        for x in self.nodes[0].p2ps:
            x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
            x.clear_last_announcement()

        tip_height = self.nodes[1].getblockcount()
        hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
        self.nodes[1].invalidateblock(hash_to_invalidate)
        all_hashes = self.nodes[1].generate(length + 1)  # Must be longer than the orig chain
        sync_blocks(self.nodes, wait=0.1)
        return [int(x, 16) for x in all_hashes]

    def run_test(self):
        # Setup the p2p connections and start up the network thread.
        inv_node = self.nodes[0].add_p2p_connection(BaseNode())
        # Set nServices to 0 for test_node, so no block download will occur outside of
        # direct fetching
        test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)

        NetworkThread().start()  # Start up network handling in another thread

        # Test logic begins here
        inv_node.wait_for_verack()
        test_node.wait_for_verack()

        # Ensure verack's have been processed by our peer
        inv_node.sync_with_ping()
        test_node.sync_with_ping()

        self.test_null_locators(test_node, inv_node)
        self.test_nonnull_locators(test_node, inv_node)

    def test_null_locators(self, test_node, inv_node):
        tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
        tip_hash = int(tip["hash"], 16)

        inv_node.check_last_announcement(inv=[tip_hash], headers=[])
        test_node.check_last_announcement(inv=[tip_hash], headers=[])

        self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
        test_node.clear_last_announcement()
        test_node.send_get_headers(locator=[], hashstop=tip_hash)
        test_node.check_last_announcement(headers=[tip_hash])

        self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
        block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
        block.solve()
        test_node.send_header_for_blocks([block])
        test_node.clear_last_announcement()
        test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
        test_node.sync_with_ping()
        assert_equal(test_node.block_announced, False)
        inv_node.clear_last_announcement()
        test_node.send_message(msg_block(block))
        inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[])
        inv_node.clear_last_announcement()

    def test_nonnull_locators(self, test_node, inv_node):
        tip = int(self.nodes[0].getbestblockhash(), 16)

        # PART 1
        # 1. Mine a block; expect inv announcements each time
        self.log.info("Part 1: headers don't start before sendheaders message...")
        for i in range(4):
            old_tip = tip
            tip = self.mine_blocks(1)
            inv_node.check_last_announcement(inv=[tip], headers=[])
            test_node.check_last_announcement(inv=[tip], headers=[])
            # Try a few different responses; none should affect next announcement
            if i == 0:
                # first request the block
                test_node.send_get_data([tip])
                test_node.wait_for_block(tip)
            elif i == 1:
{'fraction_non_alphanumeric': 0.0883192114819298, 'fraction_numerical': 0.01089399965415874, 'mean_word_length': 3.05984555984556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3285938', 'n_tokens_mistral': 6951, 'n_tokens_neox': 6653, 'n_words': 2136}