<commit_before>package main

import (
	"http"
	"io"
	"os"
	"log"
	"strings"
	"jra-go.googlecode.com/hg/linkio"
)

var gLink *linkio.Link

func init() {
	gLink = linkio.NewLink(56 /* kbps */)
}

type Proxy struct {
}

func NewProxy() *Proxy { return &Proxy{} }

func loghit(r *http.Request, code int, hit bool) {
	// hit will eventually be used to indicate if this request was a cache hit
	log.Printf("%v %v %v", r.Method, r.RawURL, code)
}

func (p *Proxy) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	if ! strings.HasPrefix(r.RawURL, "http://") {
		http.Error(wr, "501 I only proxy http", http.StatusNotImplemented)
		loghit(r, http.StatusNotImplemented, false)
		return
	}

	var resp *http.Response
	var err os.Error

	if (r.Method != "GET" && r.Method != "POST") {
		log.Print("Cannot handle method ", r.Method)
		http.Error(wr, "501 I only handle GET and POST", http.StatusNotImplemented)
		return
	}

// 	d,_ := http.DumpRequest(r, false)
//	println("dump: ", string(d))

	req := new(http.Request)
	req.NoRedirect = true

	// copy through certain headers
	req.Header = make(map[string]string)
	if ck, ok := r.Header["Cookie"]; ok {
		req.Header["Cookie"] = ck
	}

	req.URL, err = http.ParseURL(r.RawURL)

	if err != nil {
		if (r.Method == "GET") {
			resp, _, err = req.Get()
		}
		if (r.Method == "POST") {
			resp, err = req.Post(safeGetCT(r, nil, "multipart/form-data"), r.Body)
			r.Body.Close()
		}
	}

	// combined error check for GET, POST, and ParseURL
	if err != nil {
		http.Error(wr, err.String(), http.StatusInternalServerError)
		loghit(r, http.StatusInternalServerError, false)
		return
	}
	wr.SetHeader("Content-Type", safeGetCT(nil, resp, "text/plain"))
	wr.WriteHeader(resp.StatusCode)

	// simulate it coming in over gLink, a shared rate-limited link
	io.Copy(wr, gLink.NewLinkReader(resp.Body))

	resp.Body.Close()
	loghit(r, resp.StatusCode, false)
}

func safeGetCT(r1 *http.Request, r2 *http.Response, def string) (ct string) {
	var ok bool
	if r1 != nil {
		ct, ok = r1.Header["Content-Type"]
	} else {
		ct, ok = r2.Header["Content-Type"]
	}
	if ! ok {
		ct = def
	}
	return
}

func main() {
	proxy := NewProxy()
	err := http.ListenAndServe(":12345", proxy)
	if err != nil {
		log.Fatal("ListenAndServe: ", err.String())
	}
}

// ex: ts=2<commit_msg>Fix broken merge.<commit_after>package main

import (
	"http"
	"io"
	"fmt"
	"net"
	"strings"
	"log"
	"jra-go.googlecode.com/hg/linkio"
)

var gLink *linkio.Link

func init() {
	gLink = linkio.NewLink(56 /* kbps */ )
}

func loghit(r *http.Request, code int) {
	log.Printf("%v %v %v", r.Method, r.RawURL, code)
}

// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }

var doNotCopy = map[string]bool{
	"Connection":        true,
	"Transfer-Encoding": true,
	"Trailer":           true,
}

type Proxy struct{}

func NewProxy() *Proxy { return &Proxy{} }

func (p *Proxy) ServeHTTP(cwr http.ResponseWriter, creq *http.Request) {
	// c = things towards the client of the proxy
	// o = things towards origin server
	oreq := new(http.Request)
	oreq.ProtoMajor = 1
	oreq.ProtoMinor = 1
	oreq.Close = true
	oreq.Header = creq.Header
	oreq.Method = creq.Method

	ourl, err := http.ParseURL(creq.RawURL)
	if err != nil {
		http.Error(cwr, fmt.Sprint("Malformed request", err),
			http.StatusNotImplemented)
		loghit(creq, http.StatusNotImplemented)
		return
	}
	// don't set RawURL, or else the request will be written with only
	// it, instead of using the URL.Path as we want (see (* http.Request)Write)
	oreq.URL = ourl

	if oreq.URL.Scheme != "http" {
		http.Error(cwr, "I only proxy http", http.StatusNotImplemented)
		loghit(creq, http.StatusNotImplemented)
		return
	}

	if oreq.Method != "GET" && oreq.Method != "POST" {
		log.Print("Cannot handle method ", creq.Method)
		http.Error(cwr, "I only handle GET and POST", http.StatusNotImplemented)
		return
	}

	if oreq.Method == "POST" {
		oreq.Method = "POST"
		if _, ok := oreq.Header["Content-Type"]; !ok {
			oreq.Header["Content-Type"] = "multipart/form-data"
		}
		oreq.ContentLength = creq.ContentLength
		oreq.Body = creq.Body
	}

	addr := oreq.URL.Host
	if !hasPort(addr) {
		addr += ":" + oreq.URL.Scheme
	}
	c, err := net.Dial("tcp", "", addr)
	if err != nil {
		http.Error(cwr, err.String(), http.StatusGatewayTimeout)
		loghit(creq, http.StatusGatewayTimeout)
		return
	}
	c.SetReadTimeout(3 * 1e9)
	cc := http.NewClientConn(c, nil)

	// debug
	//dbg, err := http.DumpRequest(oreq, true)
	//log.Print("Dump request to origin server:\n", string(dbg))

	err = cc.Write(oreq)
	if err != nil {
		http.Error(cwr, err.String(), http.StatusGatewayTimeout)
		loghit(creq, http.StatusGatewayTimeout)
		return
	}

	oresp, err := cc.Read()
	if err != nil && err != http.ErrPersistEOF {
		http.Error(cwr, err.String(), http.StatusGatewayTimeout)
		loghit(creq, http.StatusGatewayTimeout)
		return
	}

	//dbg, err = http.DumpResponse(oresp, true)
	//log.Print("Dump response from origin server:\n", string(dbg))

	for hdr, val := range oresp.Header {
		if !doNotCopy[hdr] {
			cwr.SetHeader(hdr, val)
		}
	}
	cwr.WriteHeader(oresp.StatusCode)

	// simulate it coming in over gLink, a shared rate-limited link
	io.Copy(cwr, gLink.NewLinkReader(oresp.Body))

	cc.Close()
	c.Close()
	loghit(creq, oresp.StatusCode)
}

func main() {
	proxy := NewProxy()
	err := http.ListenAndServe(":12345", proxy)
	if err != nil {
		log.Fatal("ListenAndServe: ", err.String())
	}
}

// ex: ts=2
{'fraction_non_alphanumeric': 0.16074632221026194, 'fraction_numerical': 0.005920344456404736, 'mean_word_length': 3.6848739495798317, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2798397', 'n_tokens_mistral': 2215, 'n_tokens_neox': 2117, 'n_words': 520}