func (c *Crawler) Get(URL string) error {
	return c.request(fasthttp.MethodGet, URL, nil, nil, nil, nil)
}
Get is used to send GET requests
package predator
import (
	"context"
	"fmt"
	"net/url"
	"strings"
	"sync"
	"sync/atomic"
	"github.com/PuerkitoBio/goquery"
	"github.com/rs/zerolog"
	"github.com/thep0y/predator/cache"
	pctx "github.com/thep0y/predator/context"
	"github.com/thep0y/predator/html"
	"github.com/thep0y/predator/json"
	"github.com/tidwall/gjson"
	"github.com/valyala/fasthttp"
)
type HandleRequest func(r *Request)
type HandleResponse func(r *Response)
type HandleHTML func(he *html.HTMLElement, r *Response)
type HTMLParser struct {
	Selector string
	Handle   HandleHTML
}
type CustomRandomBoundary func() string
type Crawler struct {
	lock *sync.RWMutex
	UserAgent  string
	retryCount uint32
	retryConditions RetryConditions
	client          *fasthttp.Client
	cookies         map[string]string
	goPool          *Pool
	proxyURLPool    []string
	requestCount  uint32
	responseCount uint32
	Context context.Context
	cache cache.Cache
	cacheFields []string
	requestHandler []HandleRequest
	responseHandler []HandleResponse
	htmlHandler []*HTMLParser
	wg *sync.WaitGroup
	log zerolog.Logger
}
func NewCrawler(opts ...CrawlerOption) *Crawler {
	c := new(Crawler)
	c.UserAgent = "Predator"
	c.client = new(fasthttp.Client)
	for _, op := range opts {
		op(c)
	}
	c.lock = &sync.RWMutex{}
	c.Context = context.Background()
	capacityState := c.goPool != nil
	if capacityState {
		c.log.Info().
			Bool("state", capacityState).
			Uint64("capacity", c.goPool.capacity).
			Msg("concurrent")
	} else {
		c.log.Info().
			Bool("state", capacityState).
			Msg("concurrent")
	}
	return c
}
func (c *Crawler) Clone() *Crawler {
	return &Crawler{
		lock:            c.lock,
		UserAgent:       c.UserAgent,
		retryCount:      c.retryCount,
		retryConditions: c.retryConditions,
		client:          c.client,
		cookies:         c.cookies,
		goPool:          c.goPool,
		proxyURLPool:    c.proxyURLPool,
		Context:         c.Context,
		cache:           c.cache,
		cacheFields:     c.cacheFields,
		requestHandler:  make([]HandleRequest, 0, 5),
		responseHandler: make([]HandleResponse, 0, 5),
		htmlHandler:     make([]*HTMLParser, 0, 5),
		wg:              &sync.WaitGroup{},
		log:             c.log,
	}
}
func (c *Crawler) request(method, URL string, body []byte, cachedMap map[string]string, headers map[string]string, ctx pctx.Context) error {
	defer func() {
		if err := recover(); err != nil {
			c.log.Fatal().Err(fmt.Errorf("worker panic: %s", err))
		}
	}()
	var err error
	reqHeaders := new(fasthttp.RequestHeader)
	reqHeaders.SetMethod(method)
	u, err := url.Parse(URL)
	if err != nil {
		c.log.Error().Caller().Err(err).Send()
		return err
	}
	reqHeaders.SetRequestURI(u.RequestURI())
	reqHeaders.Set("User-Agent", c.UserAgent)
	for k, v := range headers {
		reqHeaders.Set(k, v)
	}
	if c.cookies != nil {
		for k, v := range c.cookies {
			reqHeaders.SetCookie(k, v)
		}
		c.log.Debug().
			Bytes("cookies", reqHeaders.Peek("Cookie")).
			Msg("cookies is set")
	}
	if ctx == nil {
		ctx, err = pctx.AcquireCtx()
		if err != nil {
			c.log.Error().Caller().Err(err).Send()
			return err
		}
	}
	request := AcquireRequest()
	request.URL = URL
	request.Method = method
	request.Headers = reqHeaders
	request.Ctx = ctx
	request.Body = body
	request.cachedMap = cachedMap
	request.ID = atomic.AddUint32(&c.requestCount, 1)
	request.crawler = c
	if c.goPool != nil {
		c.wg.Add(1)
		task := &Task{c, request}
		err = c.goPool.Put(task)
		if err != nil {
			c.log.Error().Caller().Err(err).Send()
			return err
		}
		return nil
	}
	err = c.prepare(request)
	if err != nil {
		return err
	}
	return nil
}
func (c *Crawler) prepare(request *Request) (err error) {
	c.log.Info().
		Uint32("request_id", atomic.LoadUint32(&request.ID)).
		Str("method", request.Method).
		Str("url", request.URL).
		Msg("requesting")
	if c.goPool != nil {
		defer c.wg.Done()
	}
	c.processRequestHandler(request)
	if request.Ctx.Length() > 0 {
		c.log.Debug().
			RawJSON("context", request.Ctx.Bytes()).
			Msg("using context")
	}
	if request.abort {
		c.log.Debug().
			Uint32("request_id", atomic.LoadUint32(&request.ID)).
			Msg("the request is aborted")
		return
	}
	var response *Response
	var key string
	if c.cache != nil {
		key, err = request.Hash()
		if err != nil {
			c.log.Error().Caller().Err(err).Send()
			return
		}
		c.log.Debug().
			Uint32("request_id", atomic.LoadUint32(&request.ID)).
			Str("cache_key", key).
			Msg("generate cache key")
		response, err = c.checkCache(key)
		if err != nil {
			return
		}
		if response != nil {
			c.log.Debug().
				Uint32("request_id", atomic.LoadUint32(&request.ID)).
				Str("cache_key", key).
				Msg("response is in the cache")
		}
	}
	var rawResp *fasthttp.Response
	if response == nil {
		response, rawResp, err = c.do(request)
		if err != nil {
			return
		}
		if c.cache != nil {
			cacheVal, err := response.Marshal()
			if err != nil {
				c.log.Error().Caller().Err(err).Send()
				return err
			}
			if cacheVal != nil {
				c.lock.Lock()
				err = c.cache.Cache(key, cacheVal)
				if err != nil {
					c.log.Error().Caller().Err(err).Send()
					return err
				}
				c.lock.Unlock()
			}
		}
	} else {
		response.Request = request
		response.Ctx = request.Ctx
	}
	c.log.Info().
		Str("method", request.Method).
		Int("status_code", response.StatusCode).
		Bool("from_cache", response.FromCache).
		Uint32("request_id", atomic.LoadUint32(&request.ID)).
		Msg("response")
	c.processResponseHandler(response)
	err = c.processHTMLHandler(response)
	if err != nil {
		return
	}
	ReleaseResponse(response)
	if rawResp != nil {
		fasthttp.ReleaseResponse(rawResp)
	}
	return
}
func (c *Crawler) checkCache(key string) (*Response, error) {
	var err error
	cachedBody, ok := c.cache.IsCached(key)
	if !ok {
		return nil, nil
	}
	var resp Response
	err = json.Unmarshal(cachedBody, &resp)
	if err != nil {
		c.log.Error().Caller().Err(err).Send()
		return nil, err
	}
	resp.FromCache = true
	return &resp, nil
}
func (c *Crawler) do(request *Request) (*Response, *fasthttp.Response, error) {
	req := fasthttp.AcquireRequest()
	req.Header = *request.Headers
	req.SetRequestURI(request.URL)
	if request.Method == fasthttp.MethodPost {
		req.SetBody(request.Body)
	}
	if request.Method == fasthttp.MethodPost && req.Header.Peek("Content-Type") == nil {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}
	if c.ProxyPoolAmount() > 0 {
		c.client.Dial = c.DialWithProxy()
	}
	if req.Header.Peek("Accept") == nil {
		req.Header.Set("Accept", "*/*")
	}
	resp := fasthttp.AcquireResponse()
	var err error
	if request.maxRedirectsCount == 0 {
		err = c.client.Do(req, resp)
	} else {
		err = c.client.DoRedirects(req, resp, int(request.maxRedirectsCount))
	}
	if err != nil {
		if p, ok := isProxyInvalid(err); ok {
			err = c.removeInvalidProxy(p)
			if err != nil {
				c.log.Fatal().Caller().Err(err).Send()
			}
			return c.do(request)
		} else {
			c.log.Fatal().Caller().Err(err).Send()
		}
	}
	atomic.AddUint32(&c.responseCount, 1)
	fasthttp.ReleaseRequest(req)
	response := &Response{
		StatusCode: resp.StatusCode(),
		Body:       resp.Body(),
		Ctx:        request.Ctx,
		Request:    request,
		Headers:    resp.Header,
	}
	if c.retryCount > 0 && request.retryCounter < c.retryCount {
		if c.retryConditions(*response) {
			atomic.AddUint32(&request.retryCounter, 1)
			c.log.Info().
				Uint32("retry_count", atomic.LoadUint32(&request.retryCounter)).
				Msg("retrying")
			return c.do(request)
		}
	}
	return response, resp, nil
}
func createBody(requestData map[string]string) []byte {
	if requestData == nil {
		return nil
	}
	form := url.Values{}
	for k, v := range requestData {
		form.Add(k, v)
	}
	return []byte(form.Encode())
}