func (f *Fetcher) visit(cmd Command, res *http.Response, err error) {
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}
	if h, ok := cmd.(Handler); ok {
		h.Handle(&Context{Cmd: cmd, Q: f.q}, res, err)
		return
	}
	f.Handler.Handle(&Context{Cmd: cmd, Q: f.q}, res, err)
}
Call the Handler for this Command. Closes the response's body.
package fetchbot
import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
	"github.com/temoto/robotstxt"
)
var (
	ErrEmptyHost = errors.New("fetchbot: invalid empty host")
	ErrDisallowed = errors.New("fetchbot: disallowed by robots.txt")
	ErrQueueClosed = errors.New("fetchbot: send on a closed queue")
)
var robotsTxtParsedPath, _ = url.Parse("/robots.txt")
const (
	DefaultCrawlDelay = 5 * time.Second
	DefaultUserAgent = "Fetchbot (https://github.com/PuerkitoBio/fetchbot)"
	DefaultWorkerIdleTTL = 30 * time.Second
)
type Doer interface {
	Do(*http.Request) (*http.Response, error)
}
type Fetcher struct {
	Handler Handler
	DisablePoliteness bool
	CrawlDelay time.Duration
	HttpClient Doer
	UserAgent string
	WorkerIdleTTL time.Duration
	AutoClose bool
	q *Queue
	dbgmu     sync.Mutex
	dbg       chan *DebugInfo
	debugging bool
	mu    sync.Mutex
	hosts map[string]chan Command
}
type DebugInfo struct {
	NumHosts int
}
func New(h Handler) *Fetcher {
	return &Fetcher{
		Handler:       h,
		CrawlDelay:    DefaultCrawlDelay,
		HttpClient:    http.DefaultClient,
		UserAgent:     DefaultUserAgent,
		WorkerIdleTTL: DefaultWorkerIdleTTL,
		dbg:           make(chan *DebugInfo, 1),
	}
}
type Queue struct {
	ch chan Command
	closed, cancelled, done chan struct{}
	wg sync.WaitGroup
}
func (q *Queue) Close() error {
	select {
	case <-q.closed:
		return nil
	default:
		close(q.closed)
		q.ch <- nil
		q.wg.Wait()
		close(q.done)
		return nil
	}
}
func (q *Queue) Block() {
	<-q.done
}
func (q *Queue) Done() <-chan struct{} {
	return q.done
}
func (q *Queue) Cancel() error {
	select {
	case <-q.cancelled:
		return nil
	default:
		close(q.cancelled)
		return q.Close()
	}
}
func (q *Queue) Send(c Command) error {
	if c == nil {
		return ErrEmptyHost
	}
	if u := c.URL(); u == nil || u.Host == "" {
		return ErrEmptyHost
	}
	select {
	case <-q.closed:
		return ErrQueueClosed
	default:
		q.ch <- c
	}
	return nil
}
func (q *Queue) SendString(method string, rawurl ...string) (int, error) {
	return q.sendWithMethod(method, rawurl)
}
func (q *Queue) SendStringHead(rawurl ...string) (int, error) {
	return q.sendWithMethod("HEAD", rawurl)
}
func (q *Queue) SendStringGet(rawurl ...string) (int, error) {
	return q.sendWithMethod("GET", rawurl)
}
func (q *Queue) sendWithMethod(method string, rawurl []string) (int, error) {
	for i, v := range rawurl {
		parsed, err := url.Parse(v)
		if err != nil {
			return i, err
		}
		if err := q.Send(&Cmd{U: parsed, M: method}); err != nil {
			return i, err
		}
	}
	return len(rawurl), nil
}
func (f *Fetcher) Start() *Queue {
	f.hosts = make(map[string]chan Command)
	f.q = &Queue{
		ch:        make(chan Command, 1),
		closed:    make(chan struct{}),
		cancelled: make(chan struct{}),
		done:      make(chan struct{}),
	}
	f.q.wg.Add(1)
	go f.processQueue()
	return f.q
}
func (f *Fetcher) Debug() <-chan *DebugInfo {
	f.dbgmu.Lock()
	defer f.dbgmu.Unlock()
	f.debugging = true
	return f.dbg
}
func (f *Fetcher) processQueue() {
loop:
	for v := range f.q.ch {
		if v == nil {
			select {
			case <-f.q.closed:
				break loop
			default:
			}
		}
		select {
		case <-f.q.cancelled:
			continue
		default:
		}
		u := v.URL()
		f.mu.Lock()
		in, ok := f.hosts[u.Host]
		if !ok {
			var rob *url.URL
			if !f.DisablePoliteness {
				rob = u.ResolveReference(robotsTxtParsedPath)
			}
			var out chan Command
			in, out = make(chan Command, 1), make(chan Command, 1)
			f.hosts[u.Host] = in
			f.mu.Unlock()
			f.q.wg.Add(1)
			go sliceIQ(in, out)
			go f.processChan(out, u.Host)
			if !f.DisablePoliteness {
				in <- robotCommand{&Cmd{U: rob, M: "GET"}}
			}
		} else {
			f.mu.Unlock()
		}
		in <- v
		f.dbgmu.Lock()
		if f.debugging {
			f.mu.Lock()
			select {
			case f.dbg <- &DebugInfo{len(f.hosts)}:
			default:
			}
			f.mu.Unlock()
		}
		f.dbgmu.Unlock()
	}
	f.mu.Lock()
	for _, ch := range f.hosts {
		close(ch)
	}
	f.hosts = make(map[string]chan Command)
	f.mu.Unlock()
	f.q.wg.Done()
}
func (f *Fetcher) processChan(ch <-chan Command, hostKey string) {
	var (
		agent *robotstxt.Group
		wait  <-chan time.Time
		ttl   <-chan time.Time
		delay = f.CrawlDelay
	)
loop:
	for {
		select {
		case <-f.q.cancelled:
			break loop
		case v, ok := <-ch:
			if !ok {
				break loop
			}
			if wait != nil {
				<-wait
			}
			select {
			case <-f.q.cancelled:
				break loop
			default:
			}
			switch r, ok := v.(robotCommand); {
			case ok:
				agent = f.getRobotAgent(r)
				if agent != nil && agent.CrawlDelay > 0 {
					delay = agent.CrawlDelay
				}
				wait = time.After(delay)
			case agent == nil || agent.Test(v.URL().Path):
				res, err := f.doRequest(v)
				f.visit(v, res, err)
				if err == nil {
					wait = time.After(delay)
				} else {
					wait = nil
				}
			default:
				f.visit(v, nil, ErrDisallowed)
				wait = nil
			}
			ttl = time.After(f.WorkerIdleTTL)
		case <-ttl:
			f.mu.Lock()
			inch, ok := f.hosts[hostKey]
			delete(f.hosts, hostKey)
			if f.AutoClose && len(f.hosts) == 0 {
				go f.q.Close()
			}
			f.mu.Unlock()
			if ok {
				close(inch)
			}
			break loop
		}
	}
	for _ = range ch {
	}
	f.q.wg.Done()
}
func (f *Fetcher) getRobotAgent(r robotCommand) *robotstxt.Group {
	res, err := f.doRequest(r)
	if err != nil {
		fmt.Fprintf(os.Stderr, "fetchbot: error fetching robots.txt: %s\n", err)
		return nil
	}
	if res.Body != nil {
		defer res.Body.Close()
	}
	robData, err := robotstxt.FromResponse(res)
	if err != nil {
		fmt.Fprintf(os.Stderr, "fetchbot: error parsing robots.txt: %s\n", err)
		return nil
	}
	return robData.FindGroup(f.UserAgent)
}