func (l *loop) finalizeTermination() {
	l.status = Stopped
	select {
	case <-l.stopC:
	default:
		close(l.stopC)
	}
	select {
	case <-l.doneC:
	default:
		close(l.doneC)
	}
	if l.sentinel != nil {
		if l.err != nil {
			l.sentinel.notifyC <- l.owner
		} else {
			l.sentinel.Forget(l)
		}
	}
	if l.err != nil {
		l.logf(true, "loop %q stopped with error: %v", l, l.err)
	} else {
		l.logf(false, "loop %q stopped", l)
	}
}
finalizeTermination notifies listeners that the loop stopped
working and a potential sentinal about its status.
package loop
import (
	"fmt"
	"sync"
	"time"
	"github.com/tideland/golib/errors"
	"github.com/tideland/golib/identifier"
	"github.com/tideland/golib/logger"
)
func Go(lf LoopFunc, dps ...interface{}) Loop {
	descr := identifier.SepIdentifier("::", dps...)
	return goLoop(lf, nil, nil, nil, descr)
}
func GoRecoverable(lf LoopFunc, rf RecoverFunc, dps ...interface{}) Loop {
	descr := identifier.SepIdentifier("::", dps...)
	return goLoop(lf, rf, nil, nil, descr)
}
func GoSentinel(dps ...interface{}) Sentinel {
	descr := identifier.SepIdentifier("::", dps...)
	return goSentinel(nil, nil, descr)
}
func GoNotifiedSentinel(nhf NotificationHandlerFunc, dps ...interface{}) Sentinel {
	descr := identifier.SepIdentifier("::", dps...)
	return goSentinel(nhf, nil, descr)
}
type Recovering struct {
	Time   time.Time
	Reason interface{}
}
type Recoverings []*Recovering
func (rs Recoverings) Frequency(num int, dur time.Duration) bool {
	if len(rs) >= num {
		first := rs[len(rs)-num].Time
		last := rs[len(rs)-1].Time
		return last.Sub(first) <= dur
	}
	return false
}
func (rs Recoverings) Len() int {
	return len(rs)
}
func (rs Recoverings) Trim(l int) Recoverings {
	if l >= len(rs) {
		return rs
	}
	return rs[len(rs)-l:]
}
func (rs Recoverings) First() *Recovering {
	if len(rs) > 0 {
		return rs[0]
	}
	return nil
}
func (rs Recoverings) Last() *Recovering {
	if len(rs) > 0 {
		return rs[len(rs)-1]
	}
	return nil
}
type RecoverFunc func(rs Recoverings) (Recoverings, error)
type Observable interface {
	fmt.Stringer
	Stop() error
	Kill(err error)
	Wait() error
	Restart() error
	Error() (status int, err error)
	attachSentinel(s *sentinel)
}
const (
	Running = iota
	Stopping
	Stopped
)
type LoopFunc func(l Loop) error
type Loop interface {
	Observable
	ShallStop() <-chan struct{}
	IsStopping() <-chan struct{}
}
type loop struct {
	mux         sync.Mutex
	descr       string
	status      int
	err         error
	loopF       LoopFunc
	recoverF    RecoverFunc
	recoverings Recoverings
	startedC    chan struct{}
	stopC       chan struct{}
	doneC       chan struct{}
	owner       Observable
	sentinel    *sentinel
}
func goLoop(lf LoopFunc, rf RecoverFunc, o Observable, s *sentinel, d string) *loop {
	l := &loop{
		descr:    d,
		loopF:    lf,
		recoverF: rf,
		startedC: make(chan struct{}),
		stopC:    make(chan struct{}),
		doneC:    make(chan struct{}),
		owner:    o,
		sentinel: s,
	}
	if l.owner == nil {
		l.owner = l
	}
	l.logf(false, "loop %q starts", l)
	go l.run()
	<-l.startedC
	return l
}
func (l *loop) String() string {
	return l.descr
}
func (l *loop) Stop() error {
	l.terminate(nil)
	return l.Wait()
}
func (l *loop) Kill(err error) {
	l.terminate(err)
}
func (l *loop) Wait() error {
	<-l.doneC
	l.mux.Lock()
	defer l.mux.Unlock()
	err := l.err
	return err
}
func (l *loop) Restart() error {
	l.mux.Lock()
	defer l.mux.Unlock()
	if l.status != Stopped {
		return errors.New(ErrRestartNonStopped, errorMessages, l)
	}
	l.err = nil
	l.recoverings = nil
	l.status = Running
	l.stopC = make(chan struct{})
	l.doneC = make(chan struct{})
	l.logf(false, "loop %q restarts", l)
	go l.run()
	<-l.startedC
	return nil
}
func (l *loop) Error() (status int, err error) {
	l.mux.Lock()
	defer l.mux.Unlock()
	status = l.status
	err = l.err
	return
}
func (l *loop) attachSentinel(s *sentinel) {
	l.mux.Lock()
	defer l.mux.Unlock()
	if l.sentinel != nil {
		l.sentinel.Forget(l)
	}
	l.sentinel = s
}
func (l *loop) ShallStop() <-chan struct{} {
	return l.stopC
}
func (l *loop) IsStopping() <-chan struct{} {
	return l.stopC
}
func (l *loop) run() {
	l.status = Running
	defer l.finalizeTermination()
	loopWrapper := func() {
		defer func() {
			if reason := recover(); reason != nil {
				l.checkTermination(reason)
			}
		}()
		l.checkTermination(l.loopF(l))
	}
	l.startedC <- struct{}{}
	for l.status == Running {
		loopWrapper()
	}
}
func (l *loop) terminate(err error) {
	l.mux.Lock()
	defer l.mux.Unlock()
	if l.err == nil {
		l.err = err
	}
	if l.status != Running {
		return
	}
	l.status = Stopping
	select {
	case <-l.stopC:
	default:
		close(l.stopC)
	}
}
func (l *loop) checkTermination(reason interface{}) {
	switch {
	case reason == nil:
		l.status = Stopping
	case l.recoverF == nil:
		l.status = Stopping
		if l.err != nil {
			break
		}
		if err, ok := reason.(error); ok {
			l.err = err
		} else {
			l.err = errors.New(ErrLoopPanicked, errorMessages, reason)
		}
	default:
		logger.Errorf("loop %q tries to recover", l)
		l.recoverings = append(l.recoverings, &Recovering{time.Now(), reason})
		l.recoverings, l.err = l.recoverF(l.recoverings)
		if l.err != nil {
			l.status = Stopping
		} else {
			l.logf(false, "loop %q recovered", l)
		}
	}
}