Make one-shot stats faster
<!--
Please make sure you've read and understood our contributing guidelines;
https://github.com/moby/moby/blob/master/CONTRIBUTING.md

** Make sure all your commits include a signature generated with `git commit -s` **

For additional information on our contributing process, read our contributing
guide https://docs.docker.com/opensource/code/

If this is a bug fix, make sure your description includes "fixes #xxxx", or
"closes #xxxx"

Please provide the following information:
-->

**- What I did**

Kuberenets CRI interface `ListContainerStats` requires to collect every pod container's stats. Currently `daemon/stats/collector.go` runs in a serial way and sleep `1s` at the end of `for` loop. This causes `ListContainerStats` to suffer long delays if there are many pod containers on the node. 

This PR is to address this issue.

**- How I did it**

- Make the stats collector parallel with the number of CPUs.
- ~`Sleep` is not needed in a normal loop since a condition variable is used. Only sleep when an error happens at system level, then retry in next loop~. `time.Sleep(s.interval)` is needed for `one-short=0` or `stream=1`, otherwise the caller would receive successive stats without any interval.
- Move `bufReader` out of `Collector`, since it cannot be shared by multiple goroutines.
- Get container stats directly (i.e. without using publishers) for `one-shot` requests. 

**- How to verify it**

Run the PR build with `cri-dockerd`, observe the time taken for `ListContainerStats`. Without this code change, it could take over 3 seconds when about 40 pod containers on the node. ~With this fix, it takes less than 1 second~. 
- Making stats collection parallel reduces the `ListContainerStats` time to 2 seconds on the 4-CPU node. 
- Getting container stats directly for `one-shot` requests reduces the `ListContainerStats` time to less than 200ms. (The requests sent by `cri-dockerd` are `one-shot` requests)  

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->
Improves the performance of the stats collector

**- A picture of a cute animal (not mandatory but encouraged)**


daemon/stats/collector.go
package stats // import "github.com/docker/docker/daemon/stats"

import (
	"bufio"
	"context"
	"sync"
	"time"

	"github.com/containerd/containerd/log"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/container"
	"github.com/docker/docker/errdefs"
	"github.com/moby/pubsub"
)

// Collector manages and provides container resource stats
type Collector struct {
	m          sync.Mutex
	cond       *sync.Cond
	supervisor supervisor
	interval   time.Duration
	publishers map[*container.Container]*pubsub.Publisher
	bufReader  *bufio.Reader
}

// NewCollector creates a stats collector that will poll the supervisor with the specified interval
func NewCollector(supervisor supervisor, interval time.Duration) *Collector {
	s := &Collector{
		interval:   interval,
		supervisor: supervisor,
		publishers: make(map[*container.Container]*pubsub.Publisher),
		bufReader:  bufio.NewReaderSize(nil, 128),
	}
	s.cond = sync.NewCond(&s.m)
	return s
}

type supervisor interface {
	// GetContainerStats collects all the stats related to a container
	GetContainerStats(container *container.Container) (*types.StatsJSON, error)
}

// Collect registers the container with the collector and adds it to
// the event loop for collection on the specified interval returning
// a channel for the subscriber to receive on.
func (s *Collector) Collect(c *container.Container) chan interface{} {
	s.cond.L.Lock()
	defer s.cond.L.Unlock()

	publisher, exists := s.publishers[c]
	if !exists {
		publisher = pubsub.NewPublisher(100*time.Millisecond, 1024)
		s.publishers[c] = publisher
	}

	s.cond.Broadcast()
	return publisher.Subscribe()
}

// StopCollection closes the channels for all subscribers and removes
// the container from metrics collection.
func (s *Collector) StopCollection(c *container.Container) {
	s.m.Lock()
	if publisher, exists := s.publishers[c]; exists {
		publisher.Close()
		delete(s.publishers, c)
	}
	s.m.Unlock()
}

// Unsubscribe removes a specific subscriber from receiving updates for a container's stats.
func (s *Collector) Unsubscribe(c *container.Container, ch chan interface{}) {
	s.m.Lock()
	publisher := s.publishers[c]
	if publisher != nil {
		publisher.Evict(ch)
		if publisher.Len() == 0 {
			delete(s.publishers, c)
		}
	}
	s.m.Unlock()
}

// Run starts the collectors and will indefinitely collect stats from the supervisor
func (s *Collector) Run() {
	type publishersPair struct {
		container *container.Container
		publisher *pubsub.Publisher
	}
	// we cannot determine the capacity here.
	// it will grow enough in first iteration
	var pairs []publishersPair

	for {
		s.cond.L.Lock()
		for len(s.publishers) == 0 {
			s.cond.Wait()
		}

		// it does not make sense in the first iteration,
		// but saves allocations in further iterations
		pairs = pairs[:0]

		for container, publisher := range s.publishers {
			// copy pointers here to release the lock ASAP
			pairs = append(pairs, publishersPair{container, publisher})
		}

		s.cond.L.Unlock()

		onlineCPUs, err := s.getNumberOnlineCPUs()
		if err != nil {
			log.G(context.TODO()).Errorf("collecting system online cpu count: %v", err)
			continue
		}

		for _, pair := range pairs {
			stats, err := s.supervisor.GetContainerStats(pair.container)

			switch err.(type) {
			case nil:
				// Sample system CPU usage close to container usage to avoid
				// noise in metric calculations.
				systemUsage, err := s.getSystemCPUUsage()
				if err != nil {
					log.G(context.TODO()).WithError(err).WithField("container_id", pair.container.ID).Errorf("collecting system cpu usage")
					continue
				}

				// FIXME: move to containerd on Linux (not Windows)
				stats.CPUStats.SystemUsage = systemUsage
				stats.CPUStats.OnlineCPUs = onlineCPUs

				pair.publisher.Publish(*stats)

			case errdefs.ErrConflict, errdefs.ErrNotFound:
				// publish empty stats containing only name and ID if not running or not found
				pair.publisher.Publish(types.StatsJSON{
					Name: pair.container.Name,
					ID:   pair.container.ID,
				})

			default:
				log.G(context.TODO()).Errorf("collecting stats for %s: %v", pair.container.ID, err)
				pair.publisher.Publish(types.StatsJSON{
					Name: pair.container.Name,
					ID:   pair.container.ID,
				})
			}
		}

		time.Sleep(s.interval)
	}
}

package stats // import "github.com/docker/docker/daemon/stats"

import (
	"sync"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/container"
	"github.com/moby/pubsub"
)

// Collector manages and provides container resource stats
type Collector struct {
	m          sync.Mutex
	cond       *sync.Cond
	supervisor supervisor
	interval   time.Duration
	publishers map[*container.Container]*pubsub.Publisher
}

// NewCollector creates a stats collector that will poll the supervisor with the specified interval
func NewCollector(supervisor supervisor, interval time.Duration) *Collector {
	s := &Collector{
		interval:   interval,
		supervisor: supervisor,
		publishers: make(map[*container.Container]*pubsub.Publisher),
	}
	s.cond = sync.NewCond(&s.m)
	return s
}

type supervisor interface {
	// GetContainerStats collects all the stats related to a container
	GetContainerStats(container *container.Container) (*types.StatsJSON, error)
}

// Collect registers the container with the collector and adds it to
// the event loop for collection on the specified interval returning
// a channel for the subscriber to receive on.
func (s *Collector) Collect(c *container.Container) chan interface{} {
	s.cond.L.Lock()
	defer s.cond.L.Unlock()

	publisher, exists := s.publishers[c]
	if !exists {
		publisher = pubsub.NewPublisher(100*time.Millisecond, 1024)
		s.publishers[c] = publisher
	}

	s.cond.Broadcast()
	return publisher.Subscribe()
}

// StopCollection closes the channels for all subscribers and removes
// the container from metrics collection.
func (s *Collector) StopCollection(c *container.Container) {
	s.m.Lock()
	if publisher, exists := s.publishers[c]; exists {
		publisher.Close()
		delete(s.publishers, c)
	}
	s.m.Unlock()
}

// Unsubscribe removes a specific subscriber from receiving updates for a container's stats.
func (s *Collector) Unsubscribe(c *container.Container, ch chan interface{}) {
	s.m.Lock()
	publisher := s.publishers[c]
	if publisher != nil {
		publisher.Evict(ch)
		if publisher.Len() == 0 {
			delete(s.publishers, c)
		}
	}
	s.m.Unlock()
}

// Run starts the collectors and will indefinitely collect stats from the supervisor
func (s *Collector) Run() {
	type publishersPair struct {
		container *container.Container
		publisher *pubsub.Publisher
	}
	// we cannot determine the capacity here.
	// it will grow enough in first iteration
	var pairs []publishersPair

	for {
		s.cond.L.Lock()
		for len(s.publishers) == 0 {
			s.cond.Wait()
		}

		// it does not make sense in the first iteration,
		// but saves allocations in further iterations
		pairs = pairs[:0]

		for container, publisher := range s.publishers {
			// copy pointers here to release the lock ASAP
			pairs = append(pairs, publishersPair{container, publisher})
		}

		s.cond.L.Unlock()

		for _, pair := range pairs {
			stats, err := s.supervisor.GetContainerStats(pair.container)
			if err != nil {
				stats = &types.StatsJSON{
					Name: pair.container.Name,
					ID:   pair.container.ID,
				}
			}
			pair.publisher.Publish(*stats)
		}

		time.Sleep(s.interval)
	}
}

Do not embed mutexes. Embedding structs makes them part of the _public_ API surface, which means that `Collector` implements `sync.Locker`: any code outside the package is free to lock and unlock the embedded mutex.