func TestBlockReplicator_3Node_Restart(t *testing.T) {
	env := createClusterEnv(t, "info", 3, nil)
	defer os.RemoveAll(env.testDir)
	require.Equal(t, 3, len(env.nodes))
	for _, node := range env.nodes {
		err := node.Start()
		require.NoError(t, err)
	}
	sameLeaderCond := func() bool {
		return env.ExistsAgreedLeader()
	}
	assert.Eventually(t, sameLeaderCond, 30*time.Second, 100*time.Millisecond)
	leaderIndex1 := env.FindLeaderIndex()
	require.True(t, leaderIndex1 >= 0)
	t.Logf("Stopping leader, index: %d", leaderIndex1)
	err := env.nodes[leaderIndex1].Close()
	require.NoError(t, err)
	isLeaderCond1 := func() bool {
		idx := env.FindLeaderIndex()
		return idx >= 0 && idx != leaderIndex1
	}
	assert.Eventually(t, isLeaderCond1, 30*time.Second, 100*time.Millisecond)
	t.Logf("Restarting node, index: %d", leaderIndex1)
	err = env.nodes[leaderIndex1].Restart()
	require.NoError(t, err)
	assert.Eventually(t, sameLeaderCond, 30*time.Second, 100*time.Millisecond)
	leaderIndex2 := env.FindLeaderIndex()
	require.True(t, leaderIndex2 >= 0)
	followerIndex := (leaderIndex2 + 1) % 3
	if followerIndex == leaderIndex1 {
		followerIndex = (leaderIndex2 + 2) % 3
	}
	t.Logf("Stopping follower, index: %d", followerIndex)
	err = env.nodes[followerIndex].Close()
	require.NoError(t, err)
	isLeaderCond2 := func() bool {
		idx := env.FindLeaderIndex()
		return idx >= 0 && idx == leaderIndex2  
	}
	assert.Eventually(t, isLeaderCond2, 30*time.Second, 100*time.Millisecond)
	t.Logf("Restarting node, index: %d", followerIndex)
	err = env.nodes[followerIndex].Restart()
	require.NoError(t, err)
	assert.Eventually(t, sameLeaderCond, 30*time.Second, 100*time.Millisecond)
	for _, node := range env.nodes {
		err := node.Close()
		require.NoError(t, err)
	}
}
Scenario:
- Start 3 nodes together, wait for leader,
- Stop the leader, wait for new leader,
- Restart the node, wait for consistent leader on all
- Stop a follower, wait for consistent leader on two,
- Restart the node, wait for consistent leader on all
package replication_test
import (
	"fmt"
	"os"
	"testing"
	"time"
	interrors "github.com/hyperledger-labs/orion-server/internal/errors"
	"github.com/hyperledger-labs/orion-server/internal/httputils"
	"github.com/hyperledger-labs/orion-server/pkg/types"
	"github.com/golang/protobuf/proto"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)
func TestBlockReplicator_3Node_StartCloseStep(t *testing.T) {
	env := createClusterEnv(t, "info", 3, nil)
	defer os.RemoveAll(env.testDir)
	require.Equal(t, 3, len(env.nodes))
	for _, node := range env.nodes {
		err := node.Start()
		require.NoError(t, err)
	}
	assert.Eventually(t, func() bool { return env.ExistsAgreedLeader() }, 30*time.Second, 100*time.Millisecond)
	leaderIndex1 := env.FindLeaderIndex()
	followerIndex1 := (leaderIndex1 + 1) % 3
	followerIndex2 := (leaderIndex1 + 1) % 3
	expectedLeaderErr := &interrors.NotLeaderError{
		LeaderID:       uint64(leaderIndex1 + 1),
		LeaderHostPort: fmt.Sprintf("127.0.0.1:%d", int(nodePortBase)+leaderIndex1+1),
	}
	for _, f := range []int{followerIndex1, followerIndex2} {
		err := env.nodes[f].blockReplicator.IsLeader()
		require.EqualError(t, err, expectedLeaderErr.Error())
	}
	require.True(t, leaderIndex1 >= 0)
	t.Logf("Leader #1 index: %d", leaderIndex1)
	err := env.nodes[leaderIndex1].Close()
	require.NoError(t, err)
	isLeaderCond := func() bool {
		idx := env.FindLeaderIndex()
		return idx >= 0 && idx != leaderIndex1
	}
	assert.Eventually(t, isLeaderCond, 30*time.Second, 100*time.Millisecond)
	leaderIndex2 := env.FindLeaderIndex()
	require.True(t, leaderIndex2 >= 0)
	t.Logf("Leader #2 index: %d", leaderIndex2)
	err = env.nodes[leaderIndex2].Close()
	require.NoError(t, err)
	var lastIndex int
	for i := range env.nodes {
		if i == leaderIndex1 || i == leaderIndex2 {
			continue
		}
		lastIndex = i
	}
	noLeaderCond := func() bool {
		err := env.nodes[lastIndex].blockReplicator.IsLeader()
		if err != nil && err.LeaderID == 0 {
			return true
		}
		return false
	}
	assert.Eventually(t, noLeaderCond, 30*time.Second, 100*time.Millisecond)
	err = env.nodes[lastIndex].Close()
	require.NoError(t, err)
}
func TestBlockReplicator_3Node_StartStepClose(t *testing.T) {
	env := createClusterEnv(t, "info", 3, nil)
	defer os.RemoveAll(env.testDir)
	require.Equal(t, 3, len(env.nodes))
	err := env.nodes[0].Start()
	require.NoError(t, err)
	noLeaderCond := func() bool {
		err := env.nodes[0].blockReplicator.IsLeader()
		if err != nil && err.LeaderID == 0 {
			return true
		}
		return false
	}
	assert.Eventually(t, noLeaderCond, 30*time.Second, 100*time.Millisecond)
	err = env.nodes[1].Start()
	require.NoError(t, err)
	assert.Eventually(t, func() bool { return env.ExistsAgreedLeader(0, 1) }, 30*time.Second, 100*time.Millisecond)
	err = env.nodes[2].Start()
	require.NoError(t, err)
	assert.Eventually(t, func() bool { return env.ExistsAgreedLeader() }, 30*time.Second, 100*time.Millisecond)
	for _, node := range env.nodes {
		err := node.Close()
		require.NoError(t, err)
	}
}