func (s *Server) getNeighborConfigs() ([]*bgpconfig.Neighbor, error) {
	var neighbors []*bgpconfig.Neighbor
	if mesh, err := s.isMeshMode(); err == nil && mesh {
		ns, err := s.getMeshNeighborConfigs()
		if err != nil {
			return nil, err
		}
		neighbors = append(neighbors, ns...)
	} else if err != nil {
		return nil, err
	}
	if ns, err := s.getGlobalNeighborConfigs(); err != nil {
		return nil, err
	} else {
		neighbors = append(neighbors, ns...)
	}
	if ns, err := s.getNodeSpecificNeighborConfigs(); err != nil {
		return nil, err
	} else {
		neighbors = append(neighbors, ns...)
	}
	return neighbors, nil
}
getNeighborConfigs returns the complete list of BGP neighbor configuration
which the node should peer.
package main
import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
	"time"
	etcd "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/pkg/transport"
	bgpapi "github.com/osrg/gobgp/api"
	bgpconfig "github.com/osrg/gobgp/config"
	bgp "github.com/osrg/gobgp/packet/bgp"
	bgpserver "github.com/osrg/gobgp/server"
	bgptable "github.com/osrg/gobgp/table"
	calicoapi "github.com/projectcalico/libcalico-go/lib/api"
	calicocli "github.com/projectcalico/libcalico-go/lib/client"
	"github.com/projectcalico/libcalico-go/lib/numorstring"
	calicoscope "github.com/projectcalico/libcalico-go/lib/scope"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
	"golang.org/x/net/context"
	"gopkg.in/tomb.v2"
)
const (
	NODENAME      = "NODENAME"
	INTERVAL      = "BGPD_INTERVAL"
	AS            = "AS"
	CALICO_PREFIX = "/calico"
	CALICO_BGP    = CALICO_PREFIX + "/bgp/v1"
	CALICO_AGGR   = CALICO_PREFIX + "/ipam/v2/host"
	CALICO_IPAM   = CALICO_PREFIX + "/v1/ipam"
	IpPoolV4       = CALICO_IPAM + "/v4/pool"
	GlobalBGP      = CALICO_BGP + "/global"
	GlobalASN      = GlobalBGP + "/as_num"
	GlobalNodeMesh = GlobalBGP + "/node_mesh"
	GlobalLogging  = GlobalBGP + "/loglevel"
	AllNodes       = CALICO_BGP + "/host"
	PollingInterval    = 300
	defaultDialTimeout = 30 * time.Second
	aggregatedPrefixSetName = "aggregated"
	hostPrefixSetName       = "host"
	RTPROT_GOBGP = 0x11
)
type IpamCache interface {
	match(string) *ipPool
	update(interface{}, bool) error
	sync() error
}
var VERSION string
func underscore(ip string) string {
	return strings.Map(func(r rune) rune {
		switch r {
		case '.', ':':
			return '_'
		}
		return r
	}, ip)
}
func errorButKeyNotFound(err error) error {
	if e, ok := err.(etcd.Error); ok && e.Code == etcd.ErrorCodeKeyNotFound {
		return nil
	}
	return err
}
func getEtcdConfig(cfg *calicoapi.CalicoAPIConfig) (etcd.Config, error) {
	var config etcd.Config
	etcdcfg := cfg.Spec.EtcdConfig
	etcdEndpoints := etcdcfg.EtcdEndpoints
	if etcdEndpoints == "" {
		etcdEndpoints = fmt.Sprintf("%s://%s", etcdcfg.EtcdScheme, etcdcfg.EtcdAuthority)
	}
	tls := transport.TLSInfo{
		CAFile:   etcdcfg.EtcdCACertFile,
		CertFile: etcdcfg.EtcdCertFile,
		KeyFile:  etcdcfg.EtcdKeyFile,
	}
	t, err := transport.NewTransport(tls, defaultDialTimeout)
	if err != nil {
		return config, err
	}
	config.Endpoints = strings.Split(etcdEndpoints, ",")
	config.Transport = t
	return config, nil
}
func recursiveNexthopLookup(bgpNexthop net.IP) (net.IP, error) {
	routes, err := netlink.RouteGet(bgpNexthop)
	if err != nil {
		return nil, err
	}
	if len(routes) == 0 {
		return nil, fmt.Errorf("no route for path: %s", bgpNexthop)
	}
	r := routes[0]
	if r.Gw != nil {
		return r.Gw, nil
	}
	return bgpNexthop, nil
}
func cleanUpRoutes() error {
	log.Println("Clean up injected routes")
	filter := &netlink.Route{
		Protocol: RTPROT_GOBGP,
	}
	list4, err := netlink.RouteListFiltered(netlink.FAMILY_V4, filter, netlink.RT_FILTER_PROTOCOL)
	if err != nil {
		return err
	}
	list6, err := netlink.RouteListFiltered(netlink.FAMILY_V6, filter, netlink.RT_FILTER_PROTOCOL)
	if err != nil {
		return err
	}
	for _, route := range append(list4, list6...) {
		netlink.RouteDel(&route)
	}
	return nil
}
type Server struct {
	t           tomb.Tomb
	bgpServer   *bgpserver.BgpServer
	datastore   calicoapi.DatastoreType
	client      *calicocli.Client
	etcd        etcd.KeysAPI
	process     *IntervalProcessor
	ipv4        net.IP
	ipv6        net.IP
	ipam        IpamCache
	reloadCh    chan []*bgptable.Path
	prefixReady chan int
}
func NewServer() (*Server, error) {
	config, err := calicocli.LoadClientConfigFromEnvironment()
	if err != nil {
		return nil, err
	}
	calicoCli, err := calicocli.New(*config)
	if err != nil {
		return nil, err
	}
	node, err := calicoCli.Nodes().Get(calicoapi.NodeMetadata{Name: os.Getenv(NODENAME)})
	if err != nil {
		return nil, err
	}
	if node.Spec.BGP == nil {
		return nil, fmt.Errorf("Calico is running in policy-only mode")
	}
	var ipv4, ipv6 net.IP
	if ipnet := node.Spec.BGP.IPv4Address; ipnet != nil {
		ipv4 = ipnet.IP
	}
	if ipnet := node.Spec.BGP.IPv6Address; ipnet != nil {
		ipv6 = ipnet.IP
	}
	bgpServer := bgpserver.NewBgpServer()
	datastoreType := config.Spec.DatastoreType
	server := Server{
		bgpServer:   bgpServer,
		datastore:   datastoreType,
		client:      calicoCli,
		ipv4:        ipv4,
		ipv6:        ipv6,
		reloadCh:    make(chan []*bgptable.Path),
		prefixReady: make(chan int),
	}
	if datastoreType == calicoapi.EtcdV2 {
		etcdConfig, err := getEtcdConfig(config)
		if err != nil {
			return nil, err
		}
		cli, err := etcd.New(etcdConfig)
		if err != nil {
			return nil, err
		}
		server.etcd = etcd.NewKeysAPI(cli)
	} else if datastoreType == calicoapi.Kubernetes {
		k8s, err := NewK8sClient(&server)
		if err != nil {
			return nil, err
		}
		ipam := NewIPAMCacheK8s(&server, server.ipamUpdateHandler)
		server.ipam = ipam
		interval := PollingInterval
		i, err := strconv.Atoi(os.Getenv(INTERVAL))
		if err == nil {
			interval = i
		}
		server.process = &IntervalProcessor{
			interval: interval,
			k8scli:   k8s,
			ipam:     ipam,
		}
	} else {
		log.Fatal("unsupported datastore type: ", datastoreType)
	}
	return &server, nil
}
func (s *Server) Serve() {
	s.t.Go(func() error {
		s.bgpServer.Serve()
		return nil
	})
	bgpAPIServer := bgpapi.NewGrpcServer(s.bgpServer, ":50051")
	s.t.Go(bgpAPIServer.Serve)
	globalConfig, err := s.getGlobalConfig()
	if err != nil {
		log.Fatal(err)
	}
	if err := s.bgpServer.Start(globalConfig); err != nil {
		log.Fatal("failed to start BGP server:", err)
	}
	if err := s.initialPolicySetting(); err != nil {
		log.Fatal(err)
	}
	if s.datastore == calicoapi.EtcdV2 {
		s.ipam = newIPAMCache(s.etcd, s.ipamUpdateHandler)
		s.t.Go(func() error { return fmt.Errorf("syncIPAM: %s", s.ipam.sync()) })
		s.t.Go(func() error { return fmt.Errorf("watchPrefix: %s", s.watchPrefix()) })
		s.t.Go(func() error { return fmt.Errorf("watchBGPConfig: %s", s.watchBGPConfig()) })
	} else if s.datastore == calicoapi.Kubernetes {
		s.t.Go(func() error { return fmt.Errorf("k8s interval loop: %s", s.process.IntervalLoop()) })
	}
	s.t.Go(func() error { return fmt.Errorf("watchBGPPath: %s", s.watchBGPPath()) })
	s.t.Go(func() error { return fmt.Errorf("watchKernelRoute: %s", s.watchKernelRoute()) })
	<-s.t.Dying()
	if err := cleanUpRoutes(); err != nil {
		log.Fatalf("%s, also failed to clean up routes which we injected: %s", s.t.Err(), err)
	}
	log.Fatal(s.t.Err())
}
func isCrossSubnet(gw net.IP, subnet net.IPNet) bool {
	p := &ipPool{CIDR: subnet.String()}
	result := !p.contain(gw.String() + "/32")
	return result
}
func (s *Server) ipamUpdateHandler(pool *ipPool) error {
	filter := &netlink.Route{
		Protocol: RTPROT_GOBGP,
	}
	list, err := netlink.RouteListFiltered(netlink.FAMILY_V4, filter, netlink.RT_FILTER_PROTOCOL)
	if err != nil {
		return err
	}
	node, err := s.client.Nodes().Get(calicoapi.NodeMetadata{Name: os.Getenv(NODENAME)})
	if err != nil {
		return err
	}
	for _, route := range list {
		if route.Dst == nil {
			continue
		}
		prefix := route.Dst.String()
		if pool.contain(prefix) {
			ipip := pool.IPIP != ""
			if pool.Mode == "cross-subnet" && !isCrossSubnet(route.Gw, node.Spec.BGP.IPv4Address.Network().IPNet) {
				ipip = false
			}
			if ipip {
				i, err := net.InterfaceByName(pool.IPIP)
				if err != nil {
					return err
				}
				route.LinkIndex = i.Index
				route.SetFlag(netlink.FLAG_ONLINK)
			} else {
				tbl, err := s.bgpServer.GetRib("", bgp.RF_IPv4_UC, []*bgptable.LookupPrefix{
					&bgptable.LookupPrefix{
						Prefix: prefix,
					},
				})
				if err != nil {
					return err
				}
				bests := tbl.Bests("")
				if len(bests) == 0 {
					log.Printf("no best for %s", prefix)
					continue
				}
				best := bests[0]
				if best.IsLocal() {
					log.Printf("%s's best is local path", prefix)
					continue
				}
				gw, err := recursiveNexthopLookup(best.GetNexthop())
				if err != nil {
					return err
				}
				route.Gw = gw
				route.Flags = 0
				rs, err := netlink.RouteGet(gw)
				if err != nil {
					return err
				}
				if len(rs) == 0 {
					return fmt.Errorf("no route for path: %s", gw)
				}
				r := rs[0]
				route.LinkIndex = r.LinkIndex
			}
			return netlink.RouteReplace(&route)
		}
	}
	return nil
}
func (s *Server) getNodeASN() (numorstring.ASNumber, error) {
	return s.getPeerASN(os.Getenv(NODENAME))
}
func (s *Server) getPeerASN(host string) (numorstring.ASNumber, error) {
	node, err := s.client.Nodes().Get(calicoapi.NodeMetadata{Name: host})
	if err != nil {
		return 0, err
	}
	if node.Spec.BGP == nil {
		return 0, fmt.Errorf("host %s is running in policy-only mode")
	}
	asn := node.Spec.BGP.ASNumber
	if asn == nil {
		return s.client.Config().GetGlobalASNumber()
	}
	return *asn, nil
}
func (s *Server) getGlobalConfig() (*bgpconfig.Global, error) {
	asn, err := s.getNodeASN()
	if err != nil {
		return nil, err
	}
	return &bgpconfig.Global{
		Config: bgpconfig.GlobalConfig{
			As:       uint32(asn),
			RouterId: s.ipv4.String(),
		},
	}, nil
}
func (s *Server) isMeshMode() (bool, error) {
	return s.client.Config().GetNodeToNodeMesh()
}
func (s *Server) getMeshNeighborConfigs() ([]*bgpconfig.Neighbor, error) {
	globalASN, err := s.getNodeASN()
	if err != nil {
		return nil, err
	}
	nodes, err := s.client.Nodes().List(calicoapi.NodeMetadata{})
	if err != nil {
		return nil, err
	}
	ns := make([]*bgpconfig.Neighbor, 0, len(nodes.Items))
	for _, node := range nodes.Items {
		if node.Metadata.Name == os.Getenv(NODENAME) {
			continue
		}
		peerASN := globalASN
		spec := node.Spec.BGP
		if spec == nil {
			continue
		}
		asn := spec.ASNumber
		if asn != nil {
			peerASN = *asn
		}
		if v4 := spec.IPv4Address; v4 != nil {
			ip := v4.IP.String()
			id := strings.Replace(ip, ".", "_", -1)
			ns = append(ns, &bgpconfig.Neighbor{
				Config: bgpconfig.NeighborConfig{
					NeighborAddress: ip,
					PeerAs:          uint32(peerASN),
					Description:     fmt.Sprintf("Mesh_%s", id),
				},
			})
		}
		if v6 := spec.IPv6Address; v6 != nil {
			ip := v6.IP.String()
			id := strings.Replace(ip, ":", "_", -1)
			ns = append(ns, &bgpconfig.Neighbor{
				Config: bgpconfig.NeighborConfig{
					NeighborAddress: ip,
					PeerAs:          uint32(peerASN),
					Description:     fmt.Sprintf("Mesh_%s", id),
				},
			})
		}
	}
	return ns, nil
}
func getNeighborConfigFromPeer(peer string, neighborType string) (*bgpconfig.Neighbor, error) {
	m := &struct {
		IP  string `json:"ip"`
		ASN string `json:"as_num"`
	}{}
	if err := json.Unmarshal([]byte(peer), m); err != nil {
		return nil, err
	}
	asn, err := numorstring.ASNumberFromString(m.ASN)
	if err != nil {
		return nil, err
	}
	return &bgpconfig.Neighbor{
		Config: bgpconfig.NeighborConfig{
			NeighborAddress: m.IP,
			PeerAs:          uint32(asn),
			Description:     fmt.Sprintf("%s_%s", strings.Title(neighborType), underscore(m.IP)),
		},
	}, nil
}
func (s *Server) getNonMeshNeighborConfigs(neighborType string) ([]*bgpconfig.Neighbor, error) {
	var metadata calicoapi.BGPPeerMetadata
	switch neighborType {
	case "global":
		metadata.Scope = calicoscope.Global
	case "node":
		metadata.Scope = calicoscope.Node
		metadata.Node = os.Getenv(NODENAME)
	default:
		return nil, fmt.Errorf("invalid neighbor type: %s", neighborType)
	}
	list, err := s.client.BGPPeers().List(metadata)
	if err != nil {
		return nil, err
	}
	ns := make([]*bgpconfig.Neighbor, 0, len(list.Items))
	for _, node := range list.Items {
		addr := node.Metadata.PeerIP.String()
		ns = append(ns, &bgpconfig.Neighbor{
			Config: bgpconfig.NeighborConfig{
				NeighborAddress: addr,
				PeerAs:          uint32(node.Spec.ASNumber),
				Description:     fmt.Sprintf("%s_%s", strings.Title(neighborType), underscore(addr)),
			},
		})
	}
	return ns, nil
}
func (s *Server) getGlobalNeighborConfigs() ([]*bgpconfig.Neighbor, error) {
	return s.getNonMeshNeighborConfigs("global")
}
func (s *Server) getNodeSpecificNeighborConfigs() ([]*bgpconfig.Neighbor, error) {
	return s.getNonMeshNeighborConfigs("node")
}