<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"errors"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

// URLCookie - cookie which stores page url
const URLCookie = "X-GoPee-URL"

// GopeeEncPrefix - encoding prefix used while encoding urls
const GopeeEncPrefix = "xox"

// Pre-compile RegEx
var reBase = regexp.MustCompile(`base +href="(.*?)"`)
var reHTML = regexp.MustCompile(`\saction=["']?(.*?)["'\s]|\shref=["']?(.*?)["'\s]|\ssrc=["']?(.*?)["'\s]`)
var reCSS = regexp.MustCompile(`url\(["']?(.*?)["']?\)`)

var reBase64 = regexp.MustCompile("^(?:[A-Za-z0-9-_]{4})*(?:[A-Za-z0-9-_]{2}==|[A-Za-z0-9-_]{3}=)?$")

var httpClient = &http.Client{}

// Hop-by-hop headers
var hopHeaders = map[string]bool{
	"connection":          true,
	"keep-alive":          true,
	"proxy-authenticate":  true,
	"proxy-authorization": true,
	"te":                true,
	"trailers":          true,
	"transfer-encoding": true,
	"upgrade":           true,
}

// Headers that create problem handling response
// TODO: support gzip compressed response in future
var problemHeaders = map[string]bool{
	"content-security-policy": true, // sent in response
	"accept-encoding":         true, // sent in request
}

type proxyManager struct {
	req  *http.Request
	uri  *url.URL
	resp *http.Response
}

func encodeURL(plainURL string) string {
	return GopeeEncPrefix + base64.URLEncoding.EncodeToString([]byte(plainURL))
}

// ProxyRequest creates a new request to be proxied
func ProxyRequest(r *http.Request, w http.ResponseWriter) {
	var uri *url.URL

	// Path can be of the form
	// /base64-encoded-string
	// /home/ajax.php

	path := strings.TrimSpace(r.URL.Path[1:])

	if strings.HasPrefix(path, GopeeEncPrefix) {
		// URL rewritten by GoPee
		path = strings.TrimLeft(path, GopeeEncPrefix)
		components := strings.Split(path, "/")
		decodedURL, err := base64.URLEncoding.DecodeString(components[0])
		if err != nil {
			log.Println(err.Error(), " for url ", path)
			return
		}
		uri, err = url.Parse(string(decodedURL[:]))
		if err != nil {
			log.Println("Error Parsing " + string(decodedURL[:]))
		}
	} else {
		// Might be a plain-text URL which was not rewritten
		urlCookie, _ := r.Cookie(URLCookie)
		// check if a cookie is set
		if urlCookie != nil {
			baseURL, err := url.Parse(urlCookie.Value)
			if err != nil {
				log.Println("Error Parsing " + urlCookie.Value)
			}
			path += "?" + r.URL.RawQuery
			pathURI, err := url.Parse(path)
			if err != nil {
				log.Println("Error Parsing " + path)
			}
			if pathURI.IsAbs() {
				uri = pathURI
			} else {
				uri = baseURL.ResolveReference(pathURI)
			}
		}
	}
	// log.Println(uri.String())
	if uri == nil {
		// return a 404
		http.NotFound(w, r)
	} else {
		// try fetching the url
		proxyMan := &proxyManager{r, uri, nil}
		proxyMan.Fetch(w)
	}
}

// Fetch makes the actual request to server and writes data with rewritten URLs to response
func (pm *proxyManager) Fetch(w http.ResponseWriter) (err error) {
	if pm.uri == nil {
		return errors.New("No URI specified to fetch")
	}
	// log.Println("Fetch: " + pm.uri.String())
	req, _ := http.NewRequest(pm.req.Method, pm.uri.String(), pm.req.Body)
	// Forward request headers, included User-Agent to server
	copyHeader(req.Header, pm.req.Header)

	pm.resp, err = httpClient.Do(req)
	if err != nil {
		log.Println("Error Fetching " + pm.uri.String())
		return err
	}
	defer pm.resp.Body.Close()

	contentType := pm.resp.Header.Get("Content-Type")

	// Forward response headers to client
	copyHeader(w.Header(), pm.resp.Header)

	// Rewrite all urls
	if strings.Contains(contentType, "text/html") {
		// HTTP is stateless, store the url in cookie to handle
		// AJAX requests for which the URLs were not rewritten
		// And don't set cookies for responses to AJAX requests
		if strings.ToLower(pm.req.Header.Get("X-Requested-With")) != "xmlhttprequest" {
			pm.uri.Fragment = ""
			// log.Println("Cookie URI ", pm.uri)
			cookie := &http.Cookie{Name: URLCookie, Value: pm.uri.String()}
			http.SetCookie(w, cookie)
		}
		pm.rewriteHTML(w)
	} else if strings.Contains(contentType, "text/css") {
		pm.rewriteCSS(w)
	} else {
		io.Copy(w, pm.resp.Body)
	}
	return nil
}

// Copy Headers from src to dst ignoring hop-by-hop headers
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		// Don't copy hop-by-hop headers and problem headers
		if hopHeaders[strings.ToLower(k)] || problemHeaders[strings.ToLower(k)] {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func (pm *proxyManager) rewriteHTML(w http.ResponseWriter) {
	body, _ := ioutil.ReadAll(pm.resp.Body)
	// if there's a <base href> specified in the document
	// use that as base to encode all URLs in the page
	baseHrefMatch := reBase.FindSubmatch(body)
	if len(baseHrefMatch) > 0 {
		var err error
		pm.uri, err = url.Parse(string(baseHrefMatch[1][:]))
		if err != nil {<commit_msg>Fix - removing existing hop-by-hop header<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"errors"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

// URLCookie - cookie which stores page url
const URLCookie = "X-GoPee-URL"

// GopeeEncPrefix - encoding prefix used while encoding urls
const GopeeEncPrefix = "xox"

// Pre-compile RegEx
var reBase = regexp.MustCompile(`base +href="(.*?)"`)
var reHTML = regexp.MustCompile(`\saction=["']?(.*?)["'\s]|\shref=["']?(.*?)["'\s]|\ssrc=["']?(.*?)["'\s]`)
var reCSS = regexp.MustCompile(`url\(["']?(.*?)["']?\)`)

var reBase64 = regexp.MustCompile("^(?:[A-Za-z0-9-_]{4})*(?:[A-Za-z0-9-_]{2}==|[A-Za-z0-9-_]{3}=)?$")

var httpClient = &http.Client{}

// Hop-by-hop headers
var hopHeaders = map[string]bool{
	"connection":          true,
	"keep-alive":          true,
	"proxy-authenticate":  true,
	"proxy-authorization": true,
	"te":                true,
	"trailers":          true,
	"transfer-encoding": true,
	"upgrade":           true,
}

// Headers that create problem handling response
// TODO: support gzip compressed response in future
var problemHeaders = map[string]bool{
	"content-security-policy": true, // sent in response
	"accept-encoding":         true, // sent in request
}

type proxyManager struct {
	req  *http.Request
	uri  *url.URL
	resp *http.Response
}

func encodeURL(plainURL string) string {
	return GopeeEncPrefix + base64.URLEncoding.EncodeToString([]byte(plainURL))
}

// ProxyRequest creates a new request to be proxied
func ProxyRequest(r *http.Request, w http.ResponseWriter) {
	var uri *url.URL

	// Path can be of the form
	// /base64-encoded-string
	// /home/ajax.php

	path := strings.TrimSpace(r.URL.Path[1:])

	if strings.HasPrefix(path, GopeeEncPrefix) {
		// URL rewritten by GoPee
		path = strings.TrimLeft(path, GopeeEncPrefix)
		components := strings.Split(path, "/")
		decodedURL, err := base64.URLEncoding.DecodeString(components[0])
		if err != nil {
			log.Println(err.Error(), " for url ", path)
			return
		}
		uri, err = url.Parse(string(decodedURL[:]))
		if err != nil {
			log.Println("Error Parsing " + string(decodedURL[:]))
		}
	} else {
		// Might be a plain-text URL which was not rewritten
		urlCookie, _ := r.Cookie(URLCookie)
		// check if a cookie is set
		if urlCookie != nil {
			baseURL, err := url.Parse(urlCookie.Value)
			if err != nil {
				log.Println("Error Parsing " + urlCookie.Value)
			}
			path += "?" + r.URL.RawQuery
			pathURI, err := url.Parse(path)
			if err != nil {
				log.Println("Error Parsing " + path)
			}
			if pathURI.IsAbs() {
				uri = pathURI
			} else {
				uri = baseURL.ResolveReference(pathURI)
			}
		}
	}
	// log.Println(uri.String())
	if uri == nil {
		// return a 404
		http.NotFound(w, r)
	} else {
		// try fetching the url
		proxyMan := &proxyManager{r, uri, nil}
		proxyMan.Fetch(w)
	}
}

// Fetch makes the actual request to server and writes data with rewritten URLs to response
func (pm *proxyManager) Fetch(w http.ResponseWriter) (err error) {
	if pm.uri == nil {
		return errors.New("No URI specified to fetch")
	}
	// log.Println("Fetch: " + pm.uri.String())
	req, _ := http.NewRequest(pm.req.Method, pm.uri.String(), pm.req.Body)
	// Forward request headers, included User-Agent to server
	copyHeader(req.Header, pm.req.Header)
	req.Proto = "HTTP/1.1"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	pm.resp, err = httpClient.Do(req)
	if err != nil {
		log.Println("Error Fetching " + pm.uri.String())
		return err
	}
	defer pm.resp.Body.Close()

	contentType := pm.resp.Header.Get("Content-Type")

	// Forward response headers to client
	copyHeader(w.Header(), pm.resp.Header)

	// Rewrite all urls
	if strings.Contains(contentType, "text/html") {
		// HTTP is stateless, store the url in cookie to handle
		// AJAX requests for which the URLs were not rewritten
		// And don't set cookies for responses to AJAX requests
		if strings.ToLower(pm.req.Header.Get("X-Requested-With")) != "xmlhttprequest" {
			pm.uri.Fragment = ""
			// log.Println("Cookie URI ", pm.uri)
			cookie := &http.Cookie{Name: URLCookie, Value: pm.uri.String()}
			http.SetCookie(w, cookie)
		}
		pm.rewriteHTML(w)
	} else if strings.Contains(contentType, "text/css") {
		pm.rewriteCSS(w)
	} else {
		io.Copy(w, pm.resp.Body)
	}
	return nil
}

// Copy Headers from src to dst ignoring hop-by-hop headers
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		// Don't copy hop-by-hop headers and problem headers
		if hopHeaders[strings.ToLower(k)] || problemHeaders[strings.ToLower(k)] {
			dst.Del(k) // remove any existing hop-by-hop header
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func (pm *proxyManager) rewriteHTML(w http.ResponseWriter) {
	body, _ := ioutil.ReadAll(pm.resp.Body)
	// if there's a <base href> specified in the document
	// use that as base to encode all URLs in the page
	baseHrefMatch := reBase.FindSubmatch(body)
	if len(baseHrefMatch) > 0 {
		var err error
		pm.uri, err = url.Parse(string(baseHrefMatch[1][:]))
		if err != nil {
{'fraction_non_alphanumeric': 0.15900918897323213, 'fraction_numerical': 0.005593288054334798, 'mean_word_length': 3.399384885764499, 'pattern_counts': {'":': 20, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6084504', 'n_tokens_mistral': 3860, 'n_tokens_neox': 3605, 'n_words': 902}