<commit_before>package ares

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"golang.org/x/net/proxy"
	"io"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/patrickmn/go-cache"
	"github.com/pborman/uuid"

	"context"
	"crypto/sha256"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"gopkg.in/olivere/elastic.v5"
	"net/url"

	"crypto/tls"
	"flag"
	"regexp"
	"rsc.io/letsencrypt"

	"github.com/PuerkitoBio/goquery"
	"path"
)

var log = logging.MustGetLogger("ares:proxy")

var (
	cachePath = flag.String("cache", "letsencrypt.cache", "cache path (default: letsencrypt.cache)")
)

type Proxy struct {
	listener net.Listener

	Hosts []Host `toml:"host"`

	Socks            string `toml:"socks"`
	ElasticsearchURL string `toml:"elasticsearch_url"`

	ListenerString    string `toml:"listener"`
	ListenerStringTLS string `toml:"tlslistener"`

	Data string `toml:"data"`

	CACertificateFile     string `toml:"ca_cert"`
	ServerCertificateFile string `toml:"server_cert"`
	ServerKeyFile         string `toml:"server_key"`
	AuthType              string `toml:"authenticationtype"`

	Logging []struct {
		Output string `toml:"output"`
		Level  string `toml:"level"`
	} `toml:"logging"`

	Cache *cache.Cache

	router *mux.Router

	index chan *Pair
	p     *Proxy
}

type Host struct {
	Host    string   `toml:"host"`
	Target  string   `toml:"target"`
	Actions []Action `toml:"action"`
}

type Action struct {
	Path       string   `toml:"path"`
	Method     []string `toml:"method"`
	RemoteAddr []string `toml:"remote_addr"`
	Location   string   `toml:"location"`
	Action     string   `toml:"action"`
	StatusCode int      `toml:"statuscode"`
	Body       string   `toml:"body"`
	UserAgent  []string `toml:"user_agent"`
	Scripts    []string `toml:"scripts"`

	File string `toml:"file"`
}

func StaticHandler(path string) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		ctype := mime.TypeByExtension(filepath.Ext(path))
		rw.Header().Set("Content-Type", ctype)

		rw.WriteHeader(http.StatusOK)

		f, err := os.Open(path)
		if err != nil {
			return
		}
		defer f.Close()

		if r.Method != "HEAD" {
			io.Copy(rw, f)
		}

		return
	}
}

type Transport struct {
	http.RoundTripper

	Proxy *Proxy
}

func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	var host *Host
	var targetURL *url.URL

	for _, h := range t.Proxy.Hosts {
		u, err := url.Parse(h.Target)
		if err != nil {
			return nil, err
		}

		hst := req.Host
		if v, _, err := net.SplitHostPort(req.Host); err == nil {
			hst = v
		}

		if u.Host != hst {
			continue
		}

		targetURL = u
		host = &h
		break
	}

	if host == nil {
		return
	}

	dump, _ := httputil.DumpRequest(req, false)
	log.Debugf("Request: %s\n\n", string(dump))

	req.Header.Del("If-Modified-Since")
	req.Header.Del("Referer")

	defer req.Body.Close()

	for name, val := range req.Header {
		for i, _ := range val {
			val[i] = strings.Replace(val[i], host.Host, targetURL.Host, -1)
		}
		req.Header[name] = val
	}

	var body []byte
	if body, err = ioutil.ReadAll(req.Body); err != nil {
		log.Errorf("Error reading body: %s", err.Error())
		return
	}<commit_msg>show error message when host not configured<commit_after>package ares

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"golang.org/x/net/proxy"
	"io"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/patrickmn/go-cache"
	"github.com/pborman/uuid"

	"context"
	"crypto/sha256"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"gopkg.in/olivere/elastic.v5"
	"net/url"

	"crypto/tls"
	"flag"
	"regexp"
	"rsc.io/letsencrypt"

	"github.com/PuerkitoBio/goquery"
	"path"
)

var log = logging.MustGetLogger("ares:proxy")

var (
	cachePath = flag.String("cache", "letsencrypt.cache", "cache path (default: letsencrypt.cache)")
)

type Proxy struct {
	listener net.Listener

	Hosts []Host `toml:"host"`

	Socks            string `toml:"socks"`
	ElasticsearchURL string `toml:"elasticsearch_url"`

	ListenerString    string `toml:"listener"`
	ListenerStringTLS string `toml:"tlslistener"`

	Data string `toml:"data"`

	CACertificateFile     string `toml:"ca_cert"`
	ServerCertificateFile string `toml:"server_cert"`
	ServerKeyFile         string `toml:"server_key"`
	AuthType              string `toml:"authenticationtype"`

	Logging []struct {
		Output string `toml:"output"`
		Level  string `toml:"level"`
	} `toml:"logging"`

	Cache *cache.Cache

	router *mux.Router

	index chan *Pair
	p     *Proxy
}

type Host struct {
	Host    string   `toml:"host"`
	Target  string   `toml:"target"`
	Actions []Action `toml:"action"`
}

type Action struct {
	Path       string   `toml:"path"`
	Method     []string `toml:"method"`
	RemoteAddr []string `toml:"remote_addr"`
	Location   string   `toml:"location"`
	Action     string   `toml:"action"`
	StatusCode int      `toml:"statuscode"`
	Body       string   `toml:"body"`
	UserAgent  []string `toml:"user_agent"`
	Scripts    []string `toml:"scripts"`

	File string `toml:"file"`
}

func StaticHandler(path string) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		ctype := mime.TypeByExtension(filepath.Ext(path))
		rw.Header().Set("Content-Type", ctype)

		rw.WriteHeader(http.StatusOK)

		f, err := os.Open(path)
		if err != nil {
			return
		}
		defer f.Close()

		if r.Method != "HEAD" {
			io.Copy(rw, f)
		}

		return
	}
}

type Transport struct {
	http.RoundTripper

	Proxy *Proxy
}

func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	var host *Host
	var targetURL *url.URL

	for _, h := range t.Proxy.Hosts {
		u, err := url.Parse(h.Target)
		if err != nil {
			return nil, err
		}

		hst := req.Host
		if v, _, err := net.SplitHostPort(req.Host); err == nil {
			hst = v
		}

		if u.Host != hst {
			continue
		}

		targetURL = u
		host = &h
		break
	}

	if host == nil {
		r, w := io.Pipe()

		resp = &http.Response{
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     make(http.Header),
			Body:       r,
			Request:    req,
			StatusCode: 404,
		}

		go func() {
			defer w.Close()
			w.Write([]byte("Host not configured."))
		}()
		return
	}

	dump, _ := httputil.DumpRequest(req, false)
	log.Debugf("Request: %s\n\n", string(dump))

	req.Header.Del("If-Modified-Since")
	req.Header.Del("Referer")

	defer req.Body.Close()

	for name, val := range req.Header {
		for i, _ := range val {
			val[i] = strings.Replace(val[i], host.Host, targetURL.Host, -1)
		}
		req.Header[name] = val
	}

	var body []byte
	if body, err = ioutil.ReadAll(req.Body); err != nil {
		log.Errorf("Error reading body: %s", err.Error())
		return
	}
{'fraction_non_alphanumeric': 0.15573529411764706, 'fraction_numerical': 0.0025, 'mean_word_length': 3.448005232177894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2901877', 'n_tokens_mistral': 2769, 'n_tokens_neox': 2577, 'n_words': 574}