func incrCacheSetStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.CacheSets) = *(StatMu.InstanceStat.CacheSets) + uint64(1)
	StatMu.Mutex.Unlock()
}
Atomically increments request delta in the instance stat
package proxy
import (
	"bytes"
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strings"
	"time"
	"github.com/DataHenHQ/datahen/pages"
	"github.com/DataHenHQ/till/internal/tillclient"
	"github.com/DataHenHQ/tillup/cache"
	"github.com/DataHenHQ/tillup/features"
	"github.com/DataHenHQ/tillup/logger"
	"github.com/DataHenHQ/tillup/sessions"
	"github.com/DataHenHQ/useragent"
	"golang.org/x/net/publicsuffix"
)
var (
	Token string
	InstanceName string
	ca       tls.Certificate
	okHeader = []byte("HTTP/1.1 200 OK\r\n\r\n")
	ForceUA = true
	UAType = "desktop"
	dhHeadersRe = regexp.MustCompile(`(?i)^X-DH`)
	ProxyFile = ""
	ProxyURLs = []string{}
	ProxyCount int
	ReleaseVersion = "dev"
	StatMu *tillclient.InstanceStatMutex
	CacheConfig cache.Config
	LoggerConfig logger.Config
	SessionsConfig sessions.Config
)
func NewPageFromRequest(r *http.Request, scheme string, pconf *PageConfig) (p *pages.Page, err error) {
	p = new(pages.Page)
	u := r.URL
	u.Host = r.Host
	u.Scheme = scheme
	p.SetURL(u.String())
	p.SetMethod(r.Method)
	nh := map[string]interface{}{}
	for name, values := range r.Header {
		nh[name] = strings.Join(values, ",")
	}
	if pconf.ForceUA {
		delete(nh, "User-Agent")
	}
	delete(nh, "Proxy-Connection")
	p.SetHeaders(nh)
	p.FetchType = "standard"
	p.UaType = pconf.UaType
	rBody, _ := ioutil.ReadAll(r.Body)
	r.Body = ioutil.NopCloser(bytes.NewReader(rBody))
	p.SetBody(string(rBody))
	p.SetUaType(pconf.UaType)
	p.SetFetchType("standard")
	p.SetPageType("default")
	gid, err := pages.GenerateGID(p)
	if err != nil {
		return nil, err
	}
	p.SetGID(gid)
	return p, nil
}
func logReqSummary(gid, method, url string, respStatus int, cachehit bool) {
	cacheType := "MISS"
	if cachehit {
		cacheType = "HIT "
	}
	fmt.Println(cacheType, gid, method, url, respStatus)
}
func sendToTarget(ctx context.Context, sconn net.Conn, sreq *http.Request, scheme string, p *pages.Page, pconf *PageConfig) (tresp *http.Response, err error) {
	var sess *sessions.Session
	if features.Allow(features.Cache) && !CacheConfig.Disabled {
		cresp, err := cache.GetResponse(ctx, p.GID, pconf.CacheFreshness, pconf.CacheServeFailures)
		if err != nil {
			return nil, err
		}
		if cresp != nil {
			incrCacheHitStatDelta()
			if sessions.IsSuccess(cresp.StatusCode) {
				incrSuccessfulRequestStatDelta()
			} else {
				incrFailedRequestStatDelta()
			}
			incrRequestStatDelta()
			logReqSummary(p.GID, sreq.Method, sreq.URL.String(), cresp.StatusCode, true)
			_, treq, terr := buildTargetRequest(scheme, sreq, pconf, sess, p)
			if terr == nil && treq != nil {
				_, tlerr := logger.StoreItem(ctx, p.GID, treq, cresp, time.Now(), true, (sessions.PageConfig)(*pconf), sess)
				if tlerr != nil {
					return nil, tlerr
				}
			}
			return cresp, nil
		}
	}
	if features.Allow(features.StickySessions) && pconf.SessionID != "" {
		sess, err = sessions.GetOrCreateStickySession(ctx, pconf.SessionID, (sessions.PageConfig)(*pconf))
		if err != nil {
			return nil, err
		}
	}
	tclient, treq, err := buildTargetRequest(scheme, sreq, pconf, sess, p)
	if err != nil {
		return nil, err
	}
	rid, tlerr := logger.StoreItem(ctx, p.GID, treq, nil, time.Now(), false, (sessions.PageConfig)(*pconf), sess)
	if tlerr != nil {
		return nil, tlerr
	}
	tresp, err = tclient.Do(treq)
	if err != nil {
		return nil, err
	}
	if sessions.IsSuccess(tresp.StatusCode) {
		incrSuccessfulRequestStatDelta()
	} else {
		incrFailedRequestStatDelta()
	}
	incrRequestStatDelta()
	if sess != nil && !sess.IsZero() {
		if pconf.StickyCookies {
			if sess.Cookies == nil {
				sess.Cookies = sessions.CookieMap{}
			}
			sess.Cookies.Set(treq.URL, tclient.Jar.Cookies(treq.URL))
		}
		sessions.SaveSession(ctx, sess)
	}
	if features.Allow(features.Cache) && !CacheConfig.Disabled {
		err := cache.StoreResponse(ctx, p.GID, tresp, nil)
		if err != nil {
			return nil, err
		}
		incrCacheSetStatDelta()
	}
	logReqSummary(p.GID, sreq.Method, sreq.URL.String(), tresp.StatusCode, false)
	tlerr = logger.UpdateItemResponse(ctx, rid, tresp, sess)
	if tlerr != nil {
		return nil, tlerr
	}
	return tresp, err
}
func buildTargetRequest(scheme string, sreq *http.Request, pconf *PageConfig, sess *sessions.Session, p *pages.Page) (*http.Client, *http.Request, error) {
	t := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		DisableCompression:    false,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		TLSHandshakeTimeout:   10 * time.Second,
		ResponseHeaderTimeout: 60 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		MaxIdleConns:          1,
		MaxIdleConnsPerHost:   1,
		IdleConnTimeout:       1 * time.Millisecond,
		MaxConnsPerHost:       1,
	}
	defer t.CloseIdleConnections()
	if pconf.UseProxy {
		var u string
		if sess != nil {
			u = sess.ProxyURL
		}
		if u == "" {
			u = getRandom(ProxyURLs)
		}
		p, err := url.Parse(u)
		if err != nil {
			return nil, nil, err
		}
		t.Proxy = http.ProxyURL(p)
	}
	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
	if err != nil {
		return nil, nil, err
	}
	tclient := &http.Client{
		Timeout:   120 * time.Second,
		Transport: t,
		Jar:       jar,
	}
	srBody, _ := ioutil.ReadAll(sreq.Body)
	sreq.Body = ioutil.NopCloser(bytes.NewReader(srBody))
	p.SetBody(string(srBody))
	treq, err := http.NewRequestWithContext(sreq.Context(), sreq.Method, sreq.RequestURI, bytes.NewReader(srBody))
	if err != nil {
		return nil, nil, err
	}
	u := sreq.URL
	u.Host = sreq.Host
	u.Scheme = scheme
	treq.URL = u
	treq.Host = u.Host
	if sess != nil && len(sess.Cookies) > 0 {
		if pconf.StickyCookies {
			tclient.Jar.SetCookies(treq.URL, sess.Cookies.Get(u))
		}
	}
	th := copySourceHeaders(sreq.Header)
	if th != nil {
		treq.Header = th
	}
	treq.Header.Del("Proxy-Connection")
	if ForceUA {
		var ua string
		if sess != nil {
			ua = sess.UserAgent
		}
		if ua == "" {
			ua, err = generateRandomUA(UAType)
			if err != nil {
				return nil, nil, err
			}
		}
		th.Set("User-Agent", ua)
	}
	return tclient, treq, nil
}
func copySourceHeaders(sh http.Header) (th http.Header) {
	th = make(http.Header)
	if sh == nil {
		return nil
	}
	for key, values := range sh {
		if dhHeadersRe.MatchString(key) {
			continue
		}
		for _, val := range values {
			th.Add(key, val)
		}
	}
	return th
}
func generateRandomUA(uaType string) (ua string, err error) {
	switch uaType {
	case "desktop":
		ua, err = useragent.Desktop()
		if err != nil {
			return "", err
		}
	case "mobile":
		ua = useragent.Mobile()
	}
	if ua == "" {
		return "", errors.New(fmt.Sprint("generated empty user agent string for", uaType))
	}
	return ua, nil
}
func writeToSource(sconn net.Conn, tresp *http.Response, p *pages.Page) (err error) {
	if p != nil {
		tresp.Header.Set("X-DH-GID", p.GetGID())
	}
	tresp.Write(sconn)
	return nil
}
func incrRequestStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.Requests) = *(StatMu.InstanceStat.Requests) + uint64(1)
	StatMu.Mutex.Unlock()
}
func incrInterceptedRequestStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.InterceptedRequests) = *(StatMu.InstanceStat.InterceptedRequests) + uint64(1)
	StatMu.Mutex.Unlock()
}
func incrFailedRequestStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.FailedRequests) = *(StatMu.InstanceStat.FailedRequests) + uint64(1)
	StatMu.Mutex.Unlock()
}
func incrSuccessfulRequestStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.SuccessfulRequests) = *(StatMu.InstanceStat.SuccessfulRequests) + uint64(1)
	StatMu.Mutex.Unlock()
}
func incrCacheHitStatDelta() {
	StatMu.Mutex.Lock()
	*(StatMu.InstanceStat.CacheHits) = *(StatMu.InstanceStat.CacheHits) + uint64(1)
	StatMu.Mutex.Unlock()
}