<commit_before>package identify_test

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p"
	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
	"github.com/libp2p/go-libp2p/p2p/protocol/identify"
	pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb"

	ic "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/event"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/peerstore"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/record"
	coretest "github.com/libp2p/go-libp2p-core/test"

	"github.com/libp2p/go-eventbus"
	blhost "github.com/libp2p/go-libp2p-blankhost"
	"github.com/libp2p/go-libp2p-peerstore/pstoremem"
	swarmt "github.com/libp2p/go-libp2p-swarm/testing"
	"github.com/libp2p/go-libp2p-testing/race"

	"github.com/libp2p/go-msgio/protoio"
	ma "github.com/multiformats/go-multiaddr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func subtestIDService(t *testing.T) {
	h1 := blhost.NewBlankHost(swarmt.GenSwarm(t))
	h2 := blhost.NewBlankHost(swarmt.GenSwarm(t))

	h1p := h1.ID()
	h2p := h2.ID()

	ids1, err := identify.NewIDService(h1)
	require.NoError(t, err)
	defer ids1.Close()

	ids2, err := identify.NewIDService(h2)
	require.NoError(t, err)
	defer ids2.Close()

	sub, err := ids1.Host.EventBus().Subscribe(new(event.EvtPeerIdentificationCompleted), eventbus.BufSize(16))
	if err != nil {
		t.Fatal(err)
	}

	testKnowsAddrs(t, h1, h2p, []ma.Multiaddr{}) // nothing
	testKnowsAddrs(t, h2, h1p, []ma.Multiaddr{}) // nothing

	// the forgetMe addr represents an address for h1 that h2 has learned out of band
	// (not via identify protocol). During the identify exchange, it will be
	// forgotten and replaced by the addrs h1 sends.
	forgetMe, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/1234")

	h2.Peerstore().AddAddr(h1p, forgetMe, peerstore.RecentlyConnectedAddrTTL)
	h2pi := h2.Peerstore().PeerInfo(h2p)
	require.NoError(t, h1.Connect(context.Background(), h2pi))

	h1t2c := h1.Network().ConnsToPeer(h2p)
	require.NotEmpty(t, h1t2c, "should have a conn here")

	ids1.IdentifyConn(h1t2c[0])

	// the idService should be opened automatically, by the network.
	// what we should see now is that both peers know about each others listen addresses.
	t.Log("test peer1 has peer2 addrs correctly")
	testKnowsAddrs(t, h1, h2p, h2.Addrs())                       // has them
	testHasCertifiedAddrs(t, h1, h2p, h2.Peerstore().Addrs(h2p)) // should have signed addrs also
	testHasProtocolVersions(t, h1, h2p)
	testHasPublicKey(t, h1, h2p, h2.Peerstore().PubKey(h2p)) // h1 should have h2's public key

	// now, this wait we do have to do. it's the wait for the Listening side
	// to be done identifying the connection.
	c := h2.Network().ConnsToPeer(h1.ID())
	require.NotEmpty(t, c, "should have connection by now at least.")
	ids2.IdentifyConn(c[0])

	// and the protocol versions.
	t.Log("test peer2 has peer1 addrs correctly")
	testKnowsAddrs(t, h2, h1p, h1.Addrs()) // has them
	testHasCertifiedAddrs(t, h2, h1p, h1.Peerstore().Addrs(h1p))
	testHasProtocolVersions(t, h2, h1p)
	testHasPublicKey(t, h2, h1p, h1.Peerstore().PubKey(h1p)) // h1 should have h2's public key

	// Need both sides to actually notice that the connection has been closed.
	h1.Network().ClosePeer(h2p)
	h2.Network().ClosePeer(h1p)
	if len(h2.Network().ConnsToPeer(h1.ID())) != 0 || len(h1.Network().ConnsToPeer(h2.ID())) != 0 {
		t.Fatal("should have no connections")
	}

	t.Log("testing addrs just after disconnect")
	// addresses don't immediately expire on disconnect, so we should still have them
	testKnowsAddrs(t, h2, h1p, h1.Addrs())
	testKnowsAddrs(t, h1, h2p, h2.Addrs())
	testHasCertifiedAddrs(t, h1, h2p, h2.Peerstore().Addrs(h2p))
	testHasCertifiedAddrs(t, h2, h1p, h1.Peerstore().Addrs(h1p))

	// the addrs had their TTLs reduced on disconnect, and
	// will be forgotten soon after
	t.Log("testing addrs after TTL expiration")
	time.Sleep(time.Second)
	testKnowsAddrs(t, h1, h2p, []ma.Multiaddr{})
	testKnowsAddrs(t, h2, h1p, []ma.Multiaddr{})
	testHasCertifiedAddrs(t, h1, h2p, []ma.Multiaddr{})
	testHasCertifiedAddrs(t, h2, h1p, []ma.Multiaddr{})

	// test that we received the "identify completed" event.
	select {
	case <-sub.Out():
	case <-time.After(3 * time.Second):
		t.Fatalf("expected EvtPeerIdentificationCompleted event within 10 seconds; none received")
	}
}

func testKnowsAddrs(t *testing.T, h host.Host, p peer.ID, expected []ma.Multiaddr) {
	t.Helper()
	assert.ElementsMatchf(t, expected, h.Peerstore().Addrs(p), fmt.Sprintf("%s did not have addr for %s", h.ID(), p))
}

func testHasCertifiedAddrs(t *testing.T, h host.Host, p peer.ID, expected []ma.Multiaddr) {
	t.Helper()
	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	if !ok {
		t.Error("expected peerstore to implement CertifiedAddrBook")
	}
	recordEnvelope := cab.GetPeerRecord(p)
	if recordEnvelope == nil {
		if len(expected) == 0 {
			return
		}
		t.Fatalf("peerstore has no signed record for peer %s", p)
	}
	r, err := recordEnvelope.Record()
	if err != nil {
		t.Error("Error unwrapping signed PeerRecord from envelope", err)
	}
	rec, ok := r.(*peer.PeerRecord)
	if !ok {
		t.Error("unexpected record type")
	}
	assert.ElementsMatchf(t, expected, rec.Addrs, fmt.Sprintf("%s did not have certified addr for %s", h.ID(), p))
}

func testHasProtocolVersions(t *testing.T, h host.Host, p peer.ID) {
	v, err := h.Peerstore().Get(p, "ProtocolVersion")
	if v == nil {
		t.Error("no protocol version")
		return
	}
	if v.(string) != identify.LibP2PVersion {
		t.Error("protocol mismatch", err)
	}
	v, err = h.Peerstore().Get(p, "AgentVersion")
	if v.(string) != "github.com/libp2p/go-libp2p" { // this is the default user agent
		t.Error("agent version mismatch", err)
	}
}

func testHasPublicKey(t *testing.T, h host.Host, p peer.ID, shouldBe ic.PubKey) {
	k := h.Peerstore().PubKey(p)
	if k == nil {
		t.Error("no public key")
		return
	}
	if !k.Equals(shouldBe) {
		t.Error("key mismatch")
		return
	}

	p2, err := peer.IDFromPublicKey(k)
	if err != nil {
		t.Error("could not make key")
	} else if p != p2 {
		t.Error("key does not match peerid")
	}
}

func getSignedRecord(t *testing.T, h host.Host, p peer.ID) *record.Envelope {
	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	require.True(t, ok)
	rec := cab.GetPeerRecord(p)
	return rec
}

// we're using BlankHost in our tests, which doesn't automatically generate peer records
// and emit address change events on the bus like BasicHost.
// This generates a record, puts it in the peerstore and emits an addr change event
// which will cause the identify service to push it to all peers it's connected to.
func emitAddrChangeEvt(t *testing.T, h host.Host) {
	t.Helper()

	key := h.Peerstore().PrivKey(h.ID())
	if key == nil {
		t.Fatal("no private key for host")
	}

	rec := peer.NewPeerRecord()
	rec.PeerID = h.ID()
	rec.Addrs = h.Addrs()
	signed, err := record.Seal(rec, key)
	if err != nil {
		t.Fatalf("error generating peer record: %s", err)
	}

	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	require.True(t, ok)
	_, err = cab.ConsumePeerRecord(signed, peerstore.PermanentAddrTTL)
	require.NoError(t, err)

	evt := event.EvtLocalAddressesUpdated{}
	emitter, err := h.EventBus().Emitter(new(event.EvtLocalAddressesUpdated), eventbus.Stateful)
	if err != nil {
		t.Fatal(err)
	}
	err = emitter.Emit(evt)
	if err != nil {
		t.Fatal(err)
	}
}

// TestIDServiceWait gives the ID service 1s to finish after dialing
// this is because it used to be concurrent. Now, Dial wait till the
// id service is done.
func TestIDService(t *testing.T) {
	// This test is highly timing dependent, waiting on timeouts/expiration.
	if race.WithRace() {
		t.Skip("skipping highly timing dependent test when race detector is enabled")
	}
	oldTTL := peerstore.RecentlyConnectedAddrTTL
	peerstore.RecentlyConnectedAddrTTL = 500 * time.Millisecond
	t.Cleanup(func() { peerstore.RecentlyConnectedAddrTTL = oldTTL })

	N := 3
	for i := 0; i < N; i++ {
		subtestIDService(t)
	}
}

func TestProtoMatching(t *testing.T) {
	tcp1, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/1234")
	tcp2, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/2345")
	tcp3, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/4567")
	utp, _ := ma.NewMultiaddr("/ip4/1.2.3.4/udp/1234/utp")

	if !identify.HasConsistentTransport(tcp1, []ma.Multiaddr{tcp2, tcp3, utp}) {
		t.Fatal("expected match")
	}

	if identify.HasConsistentTransport(utp, []ma.Multiaddr{tcp2, tcp3}) {
		t.Fatal("expected mismatch")
	}
}<commit_msg>don't repeat the TestIDService test 3 times<commit_after>package identify_test

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p"
	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
	"github.com/libp2p/go-libp2p/p2p/protocol/identify"
	pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb"

	ic "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/event"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/peerstore"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/record"
	coretest "github.com/libp2p/go-libp2p-core/test"

	"github.com/libp2p/go-eventbus"
	blhost "github.com/libp2p/go-libp2p-blankhost"
	"github.com/libp2p/go-libp2p-peerstore/pstoremem"
	swarmt "github.com/libp2p/go-libp2p-swarm/testing"
	"github.com/libp2p/go-libp2p-testing/race"

	"github.com/libp2p/go-msgio/protoio"
	ma "github.com/multiformats/go-multiaddr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func testKnowsAddrs(t *testing.T, h host.Host, p peer.ID, expected []ma.Multiaddr) {
	t.Helper()
	assert.ElementsMatchf(t, expected, h.Peerstore().Addrs(p), fmt.Sprintf("%s did not have addr for %s", h.ID(), p))
}

func testHasCertifiedAddrs(t *testing.T, h host.Host, p peer.ID, expected []ma.Multiaddr) {
	t.Helper()
	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	if !ok {
		t.Error("expected peerstore to implement CertifiedAddrBook")
	}
	recordEnvelope := cab.GetPeerRecord(p)
	if recordEnvelope == nil {
		if len(expected) == 0 {
			return
		}
		t.Fatalf("peerstore has no signed record for peer %s", p)
	}
	r, err := recordEnvelope.Record()
	if err != nil {
		t.Error("Error unwrapping signed PeerRecord from envelope", err)
	}
	rec, ok := r.(*peer.PeerRecord)
	if !ok {
		t.Error("unexpected record type")
	}
	assert.ElementsMatchf(t, expected, rec.Addrs, fmt.Sprintf("%s did not have certified addr for %s", h.ID(), p))
}

func testHasProtocolVersions(t *testing.T, h host.Host, p peer.ID) {
	v, err := h.Peerstore().Get(p, "ProtocolVersion")
	if v == nil {
		t.Error("no protocol version")
		return
	}
	if v.(string) != identify.LibP2PVersion {
		t.Error("protocol mismatch", err)
	}
	v, err = h.Peerstore().Get(p, "AgentVersion")
	if v.(string) != "github.com/libp2p/go-libp2p" { // this is the default user agent
		t.Error("agent version mismatch", err)
	}
}

func testHasPublicKey(t *testing.T, h host.Host, p peer.ID, shouldBe ic.PubKey) {
	k := h.Peerstore().PubKey(p)
	if k == nil {
		t.Error("no public key")
		return
	}
	if !k.Equals(shouldBe) {
		t.Error("key mismatch")
		return
	}

	p2, err := peer.IDFromPublicKey(k)
	if err != nil {
		t.Error("could not make key")
	} else if p != p2 {
		t.Error("key does not match peerid")
	}
}

func getSignedRecord(t *testing.T, h host.Host, p peer.ID) *record.Envelope {
	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	require.True(t, ok)
	rec := cab.GetPeerRecord(p)
	return rec
}

// we're using BlankHost in our tests, which doesn't automatically generate peer records
// and emit address change events on the bus like BasicHost.
// This generates a record, puts it in the peerstore and emits an addr change event
// which will cause the identify service to push it to all peers it's connected to.
func emitAddrChangeEvt(t *testing.T, h host.Host) {
	t.Helper()

	key := h.Peerstore().PrivKey(h.ID())
	if key == nil {
		t.Fatal("no private key for host")
	}

	rec := peer.NewPeerRecord()
	rec.PeerID = h.ID()
	rec.Addrs = h.Addrs()
	signed, err := record.Seal(rec, key)
	if err != nil {
		t.Fatalf("error generating peer record: %s", err)
	}

	cab, ok := peerstore.GetCertifiedAddrBook(h.Peerstore())
	require.True(t, ok)
	_, err = cab.ConsumePeerRecord(signed, peerstore.PermanentAddrTTL)
	require.NoError(t, err)

	evt := event.EvtLocalAddressesUpdated{}
	emitter, err := h.EventBus().Emitter(new(event.EvtLocalAddressesUpdated), eventbus.Stateful)
	if err != nil {
		t.Fatal(err)
	}
	err = emitter.Emit(evt)
	if err != nil {
		t.Fatal(err)
	}
}

// TestIDServiceWait gives the ID service 1s to finish after dialing
// this is because it used to be concurrent. Now, Dial wait till the
// id service is done.
func TestIDService(t *testing.T) {
	// This test is highly timing dependent, waiting on timeouts/expiration.
	if race.WithRace() {
		t.Skip("skipping highly timing dependent test when race detector is enabled")
	}
	oldTTL := peerstore.RecentlyConnectedAddrTTL
	peerstore.RecentlyConnectedAddrTTL = 500 * time.Millisecond
	t.Cleanup(func() { peerstore.RecentlyConnectedAddrTTL = oldTTL })

	h1 := blhost.NewBlankHost(swarmt.GenSwarm(t))
	h2 := blhost.NewBlankHost(swarmt.GenSwarm(t))

	h1p := h1.ID()
	h2p := h2.ID()

	ids1, err := identify.NewIDService(h1)
	require.NoError(t, err)
	defer ids1.Close()

	ids2, err := identify.NewIDService(h2)
	require.NoError(t, err)
	defer ids2.Close()

	sub, err := ids1.Host.EventBus().Subscribe(new(event.EvtPeerIdentificationCompleted), eventbus.BufSize(16))
	if err != nil {
		t.Fatal(err)
	}

	testKnowsAddrs(t, h1, h2p, []ma.Multiaddr{}) // nothing
	testKnowsAddrs(t, h2, h1p, []ma.Multiaddr{}) // nothing

	// the forgetMe addr represents an address for h1 that h2 has learned out of band
	// (not via identify protocol). During the identify exchange, it will be
	// forgotten and replaced by the addrs h1 sends.
	forgetMe, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/1234")

	h2.Peerstore().AddAddr(h1p, forgetMe, peerstore.RecentlyConnectedAddrTTL)
	h2pi := h2.Peerstore().PeerInfo(h2p)
	require.NoError(t, h1.Connect(context.Background(), h2pi))

	h1t2c := h1.Network().ConnsToPeer(h2p)
	require.NotEmpty(t, h1t2c, "should have a conn here")

	ids1.IdentifyConn(h1t2c[0])

	// the idService should be opened automatically, by the network.
	// what we should see now is that both peers know about each others listen addresses.
	t.Log("test peer1 has peer2 addrs correctly")
	testKnowsAddrs(t, h1, h2p, h2.Addrs())                       // has them
	testHasCertifiedAddrs(t, h1, h2p, h2.Peerstore().Addrs(h2p)) // should have signed addrs also
	testHasProtocolVersions(t, h1, h2p)
	testHasPublicKey(t, h1, h2p, h2.Peerstore().PubKey(h2p)) // h1 should have h2's public key

	// now, this wait we do have to do. it's the wait for the Listening side
	// to be done identifying the connection.
	c := h2.Network().ConnsToPeer(h1.ID())
	require.NotEmpty(t, c, "should have connection by now at least.")
	ids2.IdentifyConn(c[0])

	// and the protocol versions.
	t.Log("test peer2 has peer1 addrs correctly")
	testKnowsAddrs(t, h2, h1p, h1.Addrs()) // has them
	testHasCertifiedAddrs(t, h2, h1p, h1.Peerstore().Addrs(h1p))
	testHasProtocolVersions(t, h2, h1p)
	testHasPublicKey(t, h2, h1p, h1.Peerstore().PubKey(h1p)) // h1 should have h2's public key

	// Need both sides to actually notice that the connection has been closed.
	h1.Network().ClosePeer(h2p)
	h2.Network().ClosePeer(h1p)
	if len(h2.Network().ConnsToPeer(h1.ID())) != 0 || len(h1.Network().ConnsToPeer(h2.ID())) != 0 {
		t.Fatal("should have no connections")
	}

	t.Log("testing addrs just after disconnect")
	// addresses don't immediately expire on disconnect, so we should still have them
	testKnowsAddrs(t, h2, h1p, h1.Addrs())
	testKnowsAddrs(t, h1, h2p, h2.Addrs())
	testHasCertifiedAddrs(t, h1, h2p, h2.Peerstore().Addrs(h2p))
	testHasCertifiedAddrs(t, h2, h1p, h1.Peerstore().Addrs(h1p))

	// the addrs had their TTLs reduced on disconnect, and
	// will be forgotten soon after
	t.Log("testing addrs after TTL expiration")
	time.Sleep(time.Second)
	testKnowsAddrs(t, h1, h2p, []ma.Multiaddr{})
	testKnowsAddrs(t, h2, h1p, []ma.Multiaddr{})
	testHasCertifiedAddrs(t, h1, h2p, []ma.Multiaddr{})
	testHasCertifiedAddrs(t, h2, h1p, []ma.Multiaddr{})

	// test that we received the "identify completed" event.
	select {
	case <-sub.Out():
	case <-time.After(3 * time.Second):
		t.Fatalf("expected EvtPeerIdentificationCompleted event within 10 seconds; none received")
	}
}

func TestProtoMatching(t *testing.T) {
	tcp1, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/1234")
	tcp2, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/2345")
	tcp3, _ := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/4567")
	utp, _ := ma.NewMultiaddr("/ip4/1.2.3.4/udp/1234/utp")

	if !identify.HasConsistentTransport(tcp1, []ma.Multiaddr{tcp2, tcp3, utp}) {
		t.Fatal("expected match")
	}

	if identify.HasConsistentTransport(utp, []ma.Multiaddr{tcp2, tcp3}) {
		t.Fatal("expected mismatch")
	}
}
{'fraction_non_alphanumeric': 0.14972144846796656, 'fraction_numerical': 0.025011606313834728, 'mean_word_length': 4.245966514459665, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '646814', 'n_tokens_mistral': 6991, 'n_tokens_neox': 6750, 'n_words': 1540}