func TextContent(n *html.Node, raw bool) string {
	tc := textContentRec(n, raw)
	if !raw {
		tc = normalizeWhitespace(tc)
	}
	return tc
}
TextContent returns the full text content of n. With raw processing the
unprocessed content is returned. If raw==false then whitespace is
normalized.
package ht
import (
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
	"github.com/andybalholm/cascadia"
	"github.com/vdobler/ht/errorlist"
	"golang.org/x/net/html"
)
func init() {
	RegisterCheck(&HTMLTag{})
	RegisterCheck(&HTMLContains{})
	RegisterCheck(W3CValidHTML{})
	RegisterCheck(&Links{})
}
type W3CValidHTML struct {
	AllowedErrors int `json:",omitempty"`
	IgnoredErrors []Condition `json:",omitempty"`
}
func (w W3CValidHTML) Execute(t *Test) error {
	file := "@file:@sample.html:" + t.Response.BodyStr
	test := &Test{
		Name: "W3CValidHTML",
		Request: Request{
			Method: "POST",
			URL:    "http://validator.w3.org/nu/",
			Params: url.Values{
				"file": {file},
			},
			ParamsAs: "multipart",
			Header: http.Header{
				"Accept":     {"text/html,application/xhtml+xml"},
				"User-Agent": {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36"},
			},
			Timeout: 20 * time.Second,
		},
		Checks: CheckList{
			StatusCode{Expect: 200},
		},
	}
	time.Sleep(100 * time.Millisecond)
	err := test.Run()
	if err != nil {
		return CantCheck{err}
	}
	if test.Result.Status != Pass {
		return CantCheck{test.Result.Error}
	}
	valStat := test.Response.Response.Header.Get("X-W3C-Validator-Status")
	if valStat == "Abort" {
		return CantCheck{fmt.Errorf("Validator service sent Abort")}
	} else if valStat == "Valid" {
		return nil
	}
	doc, err := html.Parse(test.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	validationErrors := []ValidationIssue{}
	noIgnored := 0
outer:
	for _, ve := range w3cValidatorErrSel.MatchAll(doc) {
		vi := extractValidationIssue(ve)
		for _, c := range w.IgnoredErrors {
			if c.Fulfilled(vi.Message) == nil {
				noIgnored++
				continue outer
			}
		}
		validationErrors = append(validationErrors, vi)
	}
	if len(validationErrors) > w.AllowedErrors {
		errmsg := fmt.Sprintf("Ignored %d errors", noIgnored)
		for i, e := range validationErrors {
			errmsg = fmt.Sprintf("%s\nError %d:\n  %s\n  %s\n  %s",
				errmsg, i, e.Position, e.Message, e.Input)
		}
		return fmt.Errorf("%s", errmsg)
	}
	return nil
}
type ValidationIssue struct {
	Position string
	Message  string
	Input    string
}
var (
	w3cValidatorErrSel   = cascadia.MustCompile("li.error")
	w3cValidatorLocSel   = cascadia.MustCompile("p.location")
	w3cValidatorMsgSel   = cascadia.MustCompile("p span")
	w3cValidatorInputSel = cascadia.MustCompile("p.extract")
)
func extractValidationIssue(node *html.Node) ValidationIssue {
	p := TextContent(w3cValidatorLocSel.MatchFirst(node), false)
	p = strings.Replace(p, "\n     ", "", -1)
	return ValidationIssue{
		Position: p,
		Message:  TextContent(w3cValidatorMsgSel.MatchFirst(node), false),
		Input:    TextContent(w3cValidatorInputSel.MatchFirst(node), false),
	}
}
type HTMLTag struct {
	Selector string
	Count int `json:",omitempty"`
	sel cascadia.Selector
}
func (c *HTMLTag) Execute(t *Test) error {
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}
	doc, err := html.Parse(t.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	matches := c.sel.MatchAll(doc)
	switch {
	case c.Count < 0 && len(matches) > 0:
		return fmt.Errorf("Found forbidden %s", c.Selector)
	case c.Count == 0 && len(matches) == 0:
		return fmt.Errorf("Cannot find %s", c.Selector)
	case c.Count > 0:
		if cnt := len(matches); cnt != c.Count {
			return fmt.Errorf("Found %d occurrences of %s, want %d",
				cnt, c.Selector, c.Count)
		}
	}
	return nil
}
func (c *HTMLTag) Prepare(*Test) (err error) {
	c.sel, err = cascadia.Compile(c.Selector)
	if err != nil {
		c.sel = nil
		return err
	}
	return nil
}
var _ Preparable = &HTMLTag{}
type HTMLContains struct {
	Selector string
	Text []string `json:",omitempty"`
	Raw bool `json:",omitempty"`
	Complete bool `json:",omitempty"`
	InOrder bool `json:",omitempty"`
	sel cascadia.Selector
}
func (c *HTMLContains) Execute(t *Test) error {
	if c.sel == nil {
		if err := c.Prepare(t); err != nil {
			return err
		}
	}
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}
	doc, err := html.Parse(t.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	matches := c.sel.MatchAll(doc)
	if len(matches) == 0 {
		return errTagNotFound
	}
	actual := make([]string, len(matches))
	for i, m := range matches {
		actual[i] = TextContent(m, c.Raw)
	}
	last := 0
	for _, want := range c.Text {
		found := -1
		for a := last; a < len(actual); a++ {
			if want == actual[a] {
				found = a
				break
			}
		}
		if found < 0 {
			return fmt.Errorf("Missing %q, have %q", want, actual[last:])
		}
		if c.InOrder {
			last = found + 1
		} else {
			last = 0
		}
	}
	if c.Complete && len(c.Text) != len(matches) {
		return WrongCount{Got: len(matches), Want: len(c.Text)}
	}
	return nil
}
var errTagNotFound = fmt.Errorf("tag not found")
func (c *HTMLContains) Prepare(*Test) (err error) {
	c.sel, err = cascadia.Compile(c.Selector)
	if err != nil {
		c.sel = nil
		return err
	}
	return nil
}
var _ Preparable = &HTMLContains{}