<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

// Cache templates
var templates = template.Must(template.ParseFiles("home.html"))

// Pre-compile RegEx
var reBase = regexp.MustCompile("base +href=\"(.*?)\"")
var reHTML = regexp.MustCompile("src=[\"\\'](.*?)[\"\\']|href=[\"\\'](.*?)[\"\\']|action=[\"\\'](.*?)[\"\\']")
var reCSS = regexp.MustCompile("url\\([\"\\']?(.*?)[\"\\']?\\)")

var httpClient *http.Client = &http.Client{}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	// 404 for all other url path
	if r.URL.Path[1:] != "" {
		http.NotFound(w, r)
		return
	}
	r.ParseForm()
	enteredUrl := r.FormValue("url")
	if enteredUrl != "" {
		validUrl, _ := url.Parse(enteredUrl)
		// prepend http if not specified
		if validUrl.Scheme == "" {
			validUrl.Scheme = "http"
		}
		encodedUrl := base64.StdEncoding.EncodeToString([]byte(validUrl.String()))
		http.Redirect(w, r, "/p/"+encodedUrl, 302)
		return
	}
	templates.ExecuteTemplate(w, "home.html", nil)
}

func encodeURL(src []byte, baseHref string, urlString string, start int, end int) []byte {
	relURL := string(src[start:end])
	// keep anchor and javascript links intact
	if strings.Index(relURL, "#") == 0 || strings.Index(relURL, "javascript") == 0 {
		return src
	}
	// Check if url is relative and make it absolute
	if strings.Index(relURL, "http") != 0 {
		var basePath *url.URL
		if baseHref == "" {
			basePath, _ = url.Parse(urlString)
		} else {
			basePath, _ = url.Parse(baseHref)
		}
		relPath, err := url.Parse(relURL)
		if err != nil {
			return src
		}
		absURL := basePath.ResolveReference(relPath).String()
		src = bytes.Replace(src, []byte(relURL), []byte(absURL), -1)
		end = start + len(absURL)
	}
	var encodedPath []byte = make([]byte, base64.StdEncoding.EncodedLen(end-start))
	base64.StdEncoding.Encode(encodedPath, src[start:end])
	return bytes.Replace(src, src[start:end], encodedPath, -1)
}

func proxyHandler(w http.ResponseWriter, r *http.Request) {
	encodedUrl := r.URL.Path[len("/p/"):]
	url, err := base64.StdEncoding.DecodeString(encodedUrl)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	urlString := string(url[:])
	req, _ := http.NewRequest(r.Method, urlString, r.Body)
	req.Header.Set("Content-Type", r.Header.Get("Content-Type"))
	// Set request user agent to that of user's
	req.Header.Set("User-Agent", r.Header.Get("User-Agent"))

	resp, err := httpClient.Do(req)

	if err != nil {
		fmt.Println("Error Fetching " + urlString)
		http.NotFound(w, r)
		return
	}
	defer resp.Body.Close()

	contentType := resp.Header.Get("Content-Type")
	w.Header().Set("Content-Type", contentType)

	// Rewrite all urls
	baseHref := ""
	if strings.Contains(contentType, "text/html") {<commit_msg>proxy all remote resp header to client so header such as content-disposition will get set correctly.<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

// Cache templates
var templates = template.Must(template.ParseFiles("home.html"))

// Pre-compile RegEx
var reBase = regexp.MustCompile("base +href=\"(.*?)\"")
var reHTML = regexp.MustCompile("src=[\"\\'](.*?)[\"\\']|href=[\"\\'](.*?)[\"\\']|action=[\"\\'](.*?)[\"\\']")
var reCSS = regexp.MustCompile("url\\([\"\\']?(.*?)[\"\\']?\\)")

var httpClient *http.Client = &http.Client{}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	// 404 for all other url path
	if r.URL.Path[1:] != "" {
		http.NotFound(w, r)
		return
	}
	r.ParseForm()
	enteredUrl := r.FormValue("url")
	if enteredUrl != "" {
		validUrl, _ := url.Parse(enteredUrl)
		// prepend http if not specified
		if validUrl.Scheme == "" {
			validUrl.Scheme = "http"
		}
		encodedUrl := base64.StdEncoding.EncodeToString([]byte(validUrl.String()))
		http.Redirect(w, r, "/p/"+encodedUrl, 302)
		return
	}
	templates.ExecuteTemplate(w, "home.html", nil)
}

func encodeURL(src []byte, baseHref string, urlString string, start int, end int) []byte {
	relURL := string(src[start:end])
	// keep anchor and javascript links intact
	if strings.Index(relURL, "#") == 0 || strings.Index(relURL, "javascript") == 0 {
		return src
	}
	// Check if url is relative and make it absolute
	if strings.Index(relURL, "http") != 0 {
		var basePath *url.URL
		if baseHref == "" {
			basePath, _ = url.Parse(urlString)
		} else {
			basePath, _ = url.Parse(baseHref)
		}
		relPath, err := url.Parse(relURL)
		if err != nil {
			return src
		}
		absURL := basePath.ResolveReference(relPath).String()
		src = bytes.Replace(src, []byte(relURL), []byte(absURL), -1)
		end = start + len(absURL)
	}
	var encodedPath []byte = make([]byte, base64.StdEncoding.EncodedLen(end-start))
	base64.StdEncoding.Encode(encodedPath, src[start:end])
	return bytes.Replace(src, src[start:end], encodedPath, -1)
}

func proxyHandler(w http.ResponseWriter, r *http.Request) {
	encodedUrl := r.URL.Path[len("/p/"):]
	url, err := base64.StdEncoding.DecodeString(encodedUrl)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	urlString := string(url[:])
	req, _ := http.NewRequest(r.Method, urlString, r.Body)
	req.Header.Set("Content-Type", r.Header.Get("Content-Type"))
	// Set request user agent to that of user's
	req.Header.Set("User-Agent", r.Header.Get("User-Agent"))

	resp, err := httpClient.Do(req)

	if err != nil {
		fmt.Println("Error Fetching " + urlString)
		http.NotFound(w, r)
		return
	}
	defer resp.Body.Close()

	contentType := ""

	//Write all remote resp header to client
	for headerKey := range resp.Header {
		headerVal := resp.Header.Get(headerKey)
		w.Header().Set(headerKey, headerVal)
		if headerKey == "Content-Type" {
			contentType = headerVal
		}
	}

	// Rewrite all urls
	baseHref := ""
	if strings.Contains(contentType, "text/html") {
{'fraction_non_alphanumeric': 0.18980218281036834, 'fraction_numerical': 0.007503410641200546, 'mean_word_length': 3.9872448979591835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5019948', 'n_tokens_mistral': 2258, 'n_tokens_neox': 2140, 'n_words': 466}