func (c *Collector) Crawl(url string) ([]*http.Request, error) {
        if url == "" {
                return []*http.Request{}, errors.New("url was empty")
        }
        var urls sync.Map
        var subdomains sync.Map
        var jsfiles sync.Map
        var forms sync.Map
        reqsMade := &syncList{}
        c.colly.OnHTML("a[href]", c.visitHTMLFunc(&urls, &subdomains, url, reqsMade))
        if c.conf.IncludeJS || c.conf.IncludeAll {
                c.colly.OnHTML("script[src]", c.findJSFunc(&jsfiles, url, reqsMade))
        }
        if c.conf.IncludeForms || c.conf.IncludeAll {
                c.colly.OnHTML("form[action]", c.findFormsFunc(&forms, url, reqsMade))
        }
        var wg sync.WaitGroup
        wg.Add(1)
        go func() {
                defer wg.Done()
                c.parseRobots(url, reqsMade)
        }()
        wg.Add(1)
        go func() {
                defer wg.Done()
                c.parseSitemap(url, reqsMade)
        }()
        if c.conf.Wayback {
                wg.Add(1)
                go func() {
                        defer wg.Done()
                        c.visitWaybackURLs(url, &subdomains, reqsMade)
                }()
        }
        wg.Add(1)
        go func() {
                defer wg.Done()
                c.colly.Visit(url)
        }()
        wg.Wait()
        return reqsMade.Reqs, nil
}
Crawl crawls a domain for urls, subdomains, jsfiles, and forms, printing output as it goes. Returns a list of http requests made
package collector
import (
        "crypto/tls"
        "errors"
        "fmt"
        "io"
        "io/ioutil"
        "net/http"
        "net/url"
        "regexp"
        "strings"
        "sync"
        "github.com/gocolly/colly"
        "github.com/hakluke/hakrawler/pkg/config"
        sitemap "github.com/oxffaa/gopher-parse-sitemap"
        "github.com/logrusorgru/aurora"
)
type Collector struct {
        conf  *config.Config
        colly *colly.Collector
        au    aurora.Aurora
        w     io.Writer
}
func NewCollector(config *config.Config, au aurora.Aurora, w io.Writer, url string) *Collector {
        basehost, _ := parseHostFromURL(config.Url)
        hostescape := strings.ReplaceAll(basehost, ".", "\\.") 
        c := colly.NewCollector()
        switch config.Scope {
        case "www":
                c = colly.NewCollector(
                        colly.AllowedDomains(basehost, "www." + basehost),
                        colly.MaxDepth(config.Depth),
                        colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"),
                )
        case "strict":
            c = colly.NewCollector(
                    colly.AllowedDomains(basehost),
                    colly.MaxDepth(config.Depth),
                    colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"),
            )
        case "subs":
            c = colly.NewCollector(
                    colly.MaxDepth(config.Depth),
                    colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"),
                    colly.URLFilters(
                            regexp.MustCompile(".*(\\.|\\/\\/)" + hostescape + "((#|\\/|\\?).*)?"),
                    ),
            )
        default:
            c = colly.NewCollector(
                    colly.MaxDepth(config.Depth),
                    colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"),
            )
        }
        if config.Insecure {
                c.WithTransport(&http.Transport{
                    TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
                })
        }
        if config.Cookie != "" {
                c.OnRequest(func(r *colly.Request) {
                        r.Headers.Set("Cookie", config.Cookie)
                })
        }
        if config.AuthHeader != "" {
                c.OnRequest(func(r *colly.Request) {
                        r.Headers.Set("Authorization", config.AuthHeader)
                })
        }
        if config.HeadersMap != nil {
                c.OnRequest(func(r *colly.Request) {
                        for header, value := range config.HeadersMap {
                                r.Headers.Set(header, value)
                        }
                })
        }
        return &Collector{
                conf:  config,
                colly: c,
                au:    au,
                w:     w,
        }
}
type syncList struct {
        L    sync.Mutex
        Reqs []*http.Request
}
func (s *syncList) AddReq(r *http.Request) {
        s.L.Lock()
        defer s.L.Unlock()
        s.Reqs = append(s.Reqs, r)
}