<commit_before>package proxy

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/bepress/camo/decoder"
	"github.com/bepress/camo/filter"
	"github.com/reedobrien/rbp"
	"github.com/rs/zerolog"
)

const (
	// DefaultMaxSize is the maximum size we'll proxy.
	DefaultMaxSize = 5 * 1024 * 1024

	// DefaultMaxRedirects is the maximum # of redirects we'll follow.
	DefaultMaxRedirects = 10

	// DefaultRequestTimeout is the request timeout for the client.
	DefaultRequestTimeout = 4 * time.Second

	// DefaultServerName is a name to set our client and Via header to.
	DefaultServerName = "bepress/camo"

	//DefaultKABE is the default keepalives setting for Backends.
	DefaultKABE = false

	//DefaultKAFE is the default keepalives setting for Frontends.
	DefaultKAFE = false
)

// MustNew returns a Proxy handler or panics.
func MustNew(hmacKey []byte, logger zerolog.Logger, options ...func(*Proxy)) *Proxy {
	if len(hmacKey) == 0 {
		panic("hmacKey must not be nil or empty")
	}

	p := &Proxy{
		BufferPool:          rbp.NewBufferPool(),
		CheckUnicast:        true,
		Decoder:             decoder.MustNew(hmacKey),
		DisableKeepAlivesBE: DefaultKABE,
		DisableKeepAlivesFE: DefaultKAFE,
		Filter:              filter.MustNewCIDR(FilteredIPNetworks),
		LookupIP:            net.LookupIP,
		MaxRedirects:        DefaultMaxRedirects,
		MaxSize:             DefaultMaxSize,
		RequestTimeout:      DefaultRequestTimeout,
		ServerName:          DefaultServerName,

		logger: logger,
	}

	for _, opt := range options {
		opt(p)
	}

	if p.Transport == nil {

		p.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   3 * time.Second,
				KeepAlive: 30 * time.Second}).DialContext,
			TLSHandshakeTimeout: 3 * time.Second,
			MaxIdleConnsPerHost: 8,
			DisableKeepAlives:   p.DisableKeepAlivesBE,
		}
	}

	p.client = &http.Client{
		Transport: p.Transport,
		Timeout:   p.RequestTimeout,
	}
	if p.RedirFunc != nil {
		p.client.CheckRedirect = p.RedirFunc
	}

	return p
}

// ResolverFunc is the net.LookupIP signature so we can fake it in tests.
type ResolverFunc func(string) ([]net.IP, error)

// Proxy implements the handler for proxying assets.
type Proxy struct {
	BufferPool     httputil.BufferPool
	CheckUnicast   bool
	Decoder        decoder.Decoder
	Filter         *filter.CIDRFilter
	FlushInterval  time.Duration
	LookupIP       ResolverFunc
	MaxRedirects   int
	MaxSize        int64
	RedirFunc      func(*http.Request, []*http.Request) error
	RequestTimeout time.Duration
	ServerName     string
	Transport      http.RoundTripper
	client         *http.Client
	logger         zerolog.Logger

	// TODO(ro) 2017-10-02 Do we really care?
	DisableKeepAlivesBE bool
	DisableKeepAlivesFE bool
}

// ServeHTTP implements HandlerFunc.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.setResponseHeaders(w)

	if r.Method != "GET" && r.Method != "HEAD" {
		w.Header().Add("Allowed", "GET,HEAD")
		http.Error(w, fmt.Sprintf("Method not allowed: %s", r.Method), http.StatusMethodNotAllowed)
		return
	}

	if r.Header.Get("Via") == p.ServerName {
		http.Error(w, "Redirect loop detected", http.StatusNotFound)
		return
	}

	// Split path and get components.
	sig, encodedURL, err := p.splitComponents(r.URL.Path)
	if err != nil {
		// If it is not a valid signed URL, it may be a health check for the
		// ELB. Handle that here.
		if r.URL.Path == "/health" {
			fmt.Fprintln(w, "OK")
			return
		}
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// Decode the URL.
	uStr, err := p.Decoder.Decode(sig, encodedURL)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	u, err := url.Parse(uStr)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, "Invalid downstream URL: "+err.Error(), http.StatusForbidden)
		return
	}

	// Validate the target host
	if err = p.validateTarget(u); err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, "invalid host: "+err.Error(), http.StatusBadRequest)
		return
	}

	// Build the request for downstream.
	outreq, err := p.buildRequest(u, w, r)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Perform the request.
	resp, err := p.client.Do(outreq)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, fmt.Sprintf("error processing request: %q", err), http.StatusInternalServerError)
		return
	}

	// Log information about the upstream request/response.
	p.logger.Info().
		Str("upstream_domain", outreq.Host).
		Int("upstream_response", resp.StatusCode).
		Str("upstream_path", outreq.URL.Path).
		Str("content_type", resp.Header.Get("Content-Type")).
		Int64("content_length", resp.ContentLength).Msg("")

	defer resp.Body.Close()

	if resp.ContentLength > p.MaxSize {
		http.Error(w, "Payload too large", http.StatusRequestEntityTooLarge)
		return
	}

	switch resp.StatusCode {
	case 200, 304, 410:
		p.buildResponse(w, resp)
		return
	case 301, 302, 303, 307:
		http.Error(w, "Too many redirects", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		http.Error(w, "Error Fetching Resource: "+resp.Status, http.StatusBadGateway)
		return
	default:
		http.Error(w, "Unable to find suitable content", http.StatusNotFound)
		return
	}
}

// buildResponse massages the headers on the response to the upstream request.
func (p *Proxy) buildResponse(outbound http.ResponseWriter, inbound *http.Response) {
	// Remove hop-by-hop headers listed in the
	// "Connection" header of the response.
	if c := inbound.Header.Get("Connection"); c != "" {
		for _, f := range strings.Split(c, ",") {
			if f = strings.TrimSpace(f); f != "" {
				inbound.Header.Del(f)
			}
		}
	}

	for _, h := range hopHeaders {
		inbound.Header.Del(h)
	}<commit_msg>Pass 206 headers through so ranges are honored<commit_after>package proxy

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/bepress/camo/decoder"
	"github.com/bepress/camo/filter"
	"github.com/reedobrien/rbp"
	"github.com/rs/zerolog"
)

const (
	// DefaultMaxSize is the maximum size we'll proxy.
	DefaultMaxSize = 5 * 1024 * 1024

	// DefaultMaxRedirects is the maximum # of redirects we'll follow.
	DefaultMaxRedirects = 10

	// DefaultRequestTimeout is the request timeout for the client.
	DefaultRequestTimeout = 4 * time.Second

	// DefaultServerName is a name to set our client and Via header to.
	DefaultServerName = "bepress/camo"

	//DefaultKABE is the default keepalives setting for Backends.
	DefaultKABE = false

	//DefaultKAFE is the default keepalives setting for Frontends.
	DefaultKAFE = false
)

// MustNew returns a Proxy handler or panics.
func MustNew(hmacKey []byte, logger zerolog.Logger, options ...func(*Proxy)) *Proxy {
	if len(hmacKey) == 0 {
		panic("hmacKey must not be nil or empty")
	}

	p := &Proxy{
		BufferPool:          rbp.NewBufferPool(),
		CheckUnicast:        true,
		Decoder:             decoder.MustNew(hmacKey),
		DisableKeepAlivesBE: DefaultKABE,
		DisableKeepAlivesFE: DefaultKAFE,
		Filter:              filter.MustNewCIDR(FilteredIPNetworks),
		LookupIP:            net.LookupIP,
		MaxRedirects:        DefaultMaxRedirects,
		MaxSize:             DefaultMaxSize,
		RequestTimeout:      DefaultRequestTimeout,
		ServerName:          DefaultServerName,

		logger: logger,
	}

	for _, opt := range options {
		opt(p)
	}

	if p.Transport == nil {

		p.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   3 * time.Second,
				KeepAlive: 30 * time.Second}).DialContext,
			TLSHandshakeTimeout: 3 * time.Second,
			MaxIdleConnsPerHost: 8,
			DisableKeepAlives:   p.DisableKeepAlivesBE,
		}
	}

	p.client = &http.Client{
		Transport: p.Transport,
		Timeout:   p.RequestTimeout,
	}
	if p.RedirFunc != nil {
		p.client.CheckRedirect = p.RedirFunc
	}

	return p
}

// ResolverFunc is the net.LookupIP signature so we can fake it in tests.
type ResolverFunc func(string) ([]net.IP, error)

// Proxy implements the handler for proxying assets.
type Proxy struct {
	BufferPool     httputil.BufferPool
	CheckUnicast   bool
	Decoder        decoder.Decoder
	Filter         *filter.CIDRFilter
	FlushInterval  time.Duration
	LookupIP       ResolverFunc
	MaxRedirects   int
	MaxSize        int64
	RedirFunc      func(*http.Request, []*http.Request) error
	RequestTimeout time.Duration
	ServerName     string
	Transport      http.RoundTripper
	client         *http.Client
	logger         zerolog.Logger

	// TODO(ro) 2017-10-02 Do we really care?
	DisableKeepAlivesBE bool
	DisableKeepAlivesFE bool
}

// ServeHTTP implements HandlerFunc.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.setResponseHeaders(w)

	if r.Method != "GET" && r.Method != "HEAD" {
		w.Header().Add("Allowed", "GET,HEAD")
		http.Error(w, fmt.Sprintf("Method not allowed: %s", r.Method), http.StatusMethodNotAllowed)
		return
	}

	if r.Header.Get("Via") == p.ServerName {
		http.Error(w, "Redirect loop detected", http.StatusNotFound)
		return
	}

	// Split path and get components.
	sig, encodedURL, err := p.splitComponents(r.URL.Path)
	if err != nil {
		// If it is not a valid signed URL, it may be a health check for the
		// ELB. Handle that here.
		if r.URL.Path == "/health" {
			fmt.Fprintln(w, "OK")
			return
		}
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// Decode the URL.
	uStr, err := p.Decoder.Decode(sig, encodedURL)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	u, err := url.Parse(uStr)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, "Invalid downstream URL: "+err.Error(), http.StatusForbidden)
		return
	}

	// Validate the target host
	if err = p.validateTarget(u); err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, "invalid host: "+err.Error(), http.StatusBadRequest)
		return
	}

	// Build the request for downstream.
	outreq, err := p.buildRequest(u, w, r)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Perform the request.
	resp, err := p.client.Do(outreq)
	if err != nil {
		p.logger.Error().Err(err).Msg(errDetails())
		http.Error(w, fmt.Sprintf("error processing request: %q", err), http.StatusInternalServerError)
		return
	}

	// Log information about the upstream request/response.
	p.logger.Info().
		Str("upstream_domain", outreq.Host).
		Int("upstream_response", resp.StatusCode).
		Str("upstream_path", outreq.URL.Path).
		Str("content_type", resp.Header.Get("Content-Type")).
		Int64("content_length", resp.ContentLength).Msg("")

	defer resp.Body.Close()

	if resp.ContentLength > p.MaxSize {
		http.Error(w, "Payload too large", http.StatusRequestEntityTooLarge)
		return
	}

	switch resp.StatusCode {
	case 200, 206, 304, 410:
		p.buildResponse(w, resp)
		return
	case 301, 302, 303, 307:
		http.Error(w, "Too many redirects", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		http.Error(w, "Error Fetching Resource: "+resp.Status, http.StatusBadGateway)
		return
	default:
		http.Error(w, "Unable to find suitable content", http.StatusNotFound)
		return
	}
}

// buildResponse massages the headers on the response to the upstream request.
func (p *Proxy) buildResponse(outbound http.ResponseWriter, inbound *http.Response) {
	// Remove hop-by-hop headers listed in the
	// "Connection" header of the response.
	if c := inbound.Header.Get("Connection"); c != "" {
		for _, f := range strings.Split(c, ",") {
			if f = strings.TrimSpace(f); f != "" {
				inbound.Header.Del(f)
			}
		}
	}

	for _, h := range hopHeaders {
		inbound.Header.Del(h)
	}
{'fraction_non_alphanumeric': 0.12863997352746526, 'fraction_numerical': 0.010919920582395765, 'mean_word_length': 3.502420856610801, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249681', 'n_tokens_mistral': 4495, 'n_tokens_neox': 4116, 'n_words': 1044}