func (h *HttpFetcher) normaliseUrl(parent *url.URL, urlString string) *url.URL {
	i := strings.LastIndex(urlString, "#")
	if i >= 0 {
		urlString = urlString[:i]
	}
	uri, err := url.Parse(urlString)
	if err != nil {
		log.Debugf("Failed to parse URL: %s", urlString)
		return nil
	}
	abs := parent.ResolveReference(uri)
	return abs
}
normaliseUrl converts relative URLs to absolute URLs
package main
import (
	"errors"
	"net/url"
	"strings"
	html "code.google.com/p/go.net/html"
	atom "code.google.com/p/go.net/html/atom"
	"github.com/PuerkitoBio/goquery"
	log "github.com/cihub/seelog"
)
var (
	InvalidNode                 = errors.New("Node is not an anchor")
	InvalidNodeAttributeMissing = errors.New("Node does not contain the specified attribute")
)
type Fetcher interface {
	Fetch(target *url.URL) (urls []*url.URL, assets []*url.URL, err error)
}
type HttpFetcher struct{}
func (h *HttpFetcher) Fetch(target *url.URL) ([]*url.URL, []*url.URL, error) {
	doc, err := goquery.NewDocument(target.String())
	if err != nil {
		return nil, nil, err
	}
	urls, err := h.extractLinks(doc)
	if err != nil {
		return nil, nil, err
	}
	assets, err := h.extractAssets(doc)
	if err != nil {
		return nil, nil, err
	}
	log.Debugf("URLs: %+v", urls)
	log.Debugf("Assets: %+v", assets)
	return urls, assets, nil
}
func (h *HttpFetcher) extractLinks(doc *goquery.Document) ([]*url.URL, error) {
	urls := make([]*url.URL, 0)
	sel := doc.Find("a")
	if sel == nil {
		return nil, nil
	}
	for _, n := range sel.Nodes {
		href, err := h.extractValidHref(n)
		if err != nil || href == "" {
			continue
		}
		if uri := h.normaliseUrl(doc.Url, href); uri != nil {
			urls = append(urls, uri)
		}
	}
	return h.dedupeUrls(urls), nil
}
func (h *HttpFetcher) extractAssets(doc *goquery.Document) ([]*url.URL, error) {
	var sel *goquery.Selection
	assets := make([]*url.URL, 0)
	sel = doc.Find("img")
	for _, n := range sel.Nodes {
		if n == nil {
			continue
		}
		for _, a := range n.Attr {
			if a.Key == "src" && a.Val != "" {
				if uri := h.normaliseUrl(doc.Url, a.Val); uri != nil {
					assets = append(assets, uri)
					break
				}
			}
		}
	}
	sel = doc.Find("script")
	for _, n := range sel.Nodes {
		if n == nil {
			continue
		}
		for _, a := range n.Attr {
			if a.Key == "src" && a.Val != "" {
				if uri := h.normaliseUrl(doc.Url, a.Val); uri != nil {
					assets = append(assets, uri)
					break
				}
			}
		}
	}
	sel = doc.Find("link")
	for _, n := range sel.Nodes {
		if n == nil {
			continue
		}
		var rel, linktype string
		var uri *url.URL
		for _, a := range n.Attr {
			switch a.Key {
			case "rel":
				rel = a.Val
			case "type":
				linktype = a.Val
			case "href":
				uri = h.normaliseUrl(doc.Url, a.Val)
			}
		}
		if uri == nil {
			continue
		}
		switch {
		case rel == "stylesheet" && linktype == "text/css":
			assets = append(assets, uri)
		case rel == "shortcut icon":
			assets = append(assets, uri)
		}
	}
	return h.dedupeUrls(assets), nil
}
func (h *HttpFetcher) extractValidHref(n *html.Node) (string, error) {
	var href string
	if n == nil || n.Type != html.ElementNode || n.DataAtom != atom.A {
		return href, InvalidNode
	}
	for _, a := range n.Attr {
		if a.Key == "href" && a.Val != "" {
			return a.Val, nil
		}
	}
	return "", InvalidNodeAttributeMissing
}