<commit_before>package main

import (
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

type Proxy struct {
	config           Config
	contentTypeRegex *regexp.Regexp
}

func newProxy(config Config) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.allowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if params.Get("url") != "" {
		p.proxyRequest(w, params)
	} else {
		log.Println("No request url to proxy")
		http.Error(w, "No request url to proxy", http.StatusBadRequest)
	}
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	url := params.Get("url")
	resp, err := http.Get(url) // http.Get follows up to 10 redirects
	if err != nil {
		log.Print(err)
		// Todo, handle specific errors
		http.Error(w, "Could not proxy", http.StatusInternalServerError)
	}

	if p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		p.writeResponse(w, resp)
	} else {
		log.Println("Upstream content doesn't match configured allowedContentTypes")
		http.Error(w, "Upstream content doesn't match configured allowedContentTypes", http.StatusBadRequest)
	}
}

func (p Proxy) writeResponse(w http.ResponseWriter, resp *http.Response) {
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	io.Copy(w, resp.Body)
	resp.Body.Close()
}
<commit_msg>Pull out request checks into validRequest helper function :lipstick:<commit_after>package main

import (
	"errors"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

type Proxy struct {
	config           Config
	contentTypeRegex *regexp.Regexp
}

func newProxy(config Config) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.allowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if err := p.validRequest(params); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
	} else {
		p.proxyRequest(w, params)
	}
}

func (p Proxy) validRequest(params url.Values) error {
	url := params.Get("url")
	if url == "" {
		return errors.New("No request url to proxy")
	}

	return nil
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	url := params.Get("url")
	resp, err := http.Get(url) // http.Get follows up to 10 redirects
	if err != nil {
		log.Print(err)
		// Todo, handle specific errors
		http.Error(w, "Could not proxy", http.StatusInternalServerError)
	}

	if p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		p.writeResponse(w, resp)
	} else {
		log.Println("Upstream content doesn't match configured allowedContentTypes")
		http.Error(w, "Upstream content doesn't match configured allowedContentTypes", http.StatusBadRequest)
	}
}

func (p Proxy) writeResponse(w http.ResponseWriter, resp *http.Response) {
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	io.Copy(w, resp.Body)
	resp.Body.Close()
}

{'fraction_non_alphanumeric': 0.13811297330850403, 'fraction_numerical': 0.0012414649286157666, 'mean_word_length': 4.140350877192983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7329557', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1107, 'n_words': 272}