<commit_before>package main

import (
	"errors"
	"github.com/elazarl/goproxy"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

type StubResolver struct {
	FileChecker        FileChecker
	UseHostAndProtocol bool
	StubRoot           string
}

func (sr StubResolver) ReturnFileResponse() ResponseGenerator {
	return func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		var resolvedFilename = constructFilename(r.Proto, r.Host, r.URL.Path, r.Method, sr.UseHostAndProtocol, sr.StubRoot)

		var fileContent, _ = ioutil.ReadFile(resolvedFilename)
		fileContentString := string(fileContent[:])
		statusCode := resolveStatusCode(fileContentString)

		fileContentString = stripMetaData(fileContentString)

		return r, goproxy.NewResponse(r, "application/json", statusCode, fileContentString)
	}
}

func (sr StubResolver) CheckFilesystemForRequest() goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		var filename = constructFilename(req.Proto, req.Host, req.URL.Path, req.Method, sr.UseHostAndProtocol, sr.StubRoot)
		if _, err := sr.FileChecker(filename); err == nil {
			return true
		}
		return false
	}
}

type ResponseGenerator func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response)

func constructFilename(proto string, host string, reqPath string, method string, useHostAndProtocol bool, basePath string) string {
	slash := string(os.PathSeparator)
	verb := strings.ToLower(method)
	protocol := strings.ToLower(strings.Split(proto, "/")[0])
	urlpath := reqPath
	if strings.HasSuffix(urlpath, "/") {
		urlpath = urlpath + "index"
	} else {
		urlpath = "/_" + strings.TrimLeft(urlpath, "/")
	}

	hostfilename := path.Join(basePath, path.Clean("."+slash+protocol+slash+host+slash+urlpath+"."+verb+".json"))
	filename := path.Join(basePath, path.Clean("."+slash+urlpath+"."+verb+".json"))

	if useHostAndProtocol {
		return hostfilename
	} else {
		return filename
	}
}

type FileChecker func(name string) (fi os.FileInfo, err error)

func resolveStatusCode(s string) int {
	regex := regexp.MustCompile(`//!.+statusCode:\s+(\d{3})`)
	result := regex.FindStringSubmatch(s)
	if len(result) > 0 {
		intresult, _ := strconv.Atoi(result[1])
		return intresult
	}
	return 200
}

func stripMetaData(s string) string {
	if strings.HasPrefix(s, "//!") {
		stringarray := strings.SplitAfterN(s, "\n", 2)
		return stringarray[1]
	}
	return s
}

type ResponseFunc func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response

func (sr StubResolver) RecordResponse() ResponseFunc {
	return func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		var filename = constructFilename(ctx.Req.Proto, ctx.Req.Host, ctx.Req.URL.Path, ctx.Req.Method, sr.UseHostAndProtocol, sr.StubRoot)
		sr.saveResponse(resp, ctx, filename)
		return resp
	}
}

func (sr StubResolver) saveResponse(resp *http.Response, ctx *goproxy.ProxyCtx, filename string) {
	if resp != nil {
		//TODO Cleanup error
		ctx.Logf("Writing response to here: " + filename)
		if _, err := sr.FileChecker(filepath.Dir(filename)); err != nil {
			os.MkdirAll(filepath.Dir(filename), 0600)
		}
		f, _ := os.Create(filename)
		defer f.Close()

		resp.Body = NewTeeReadCloser(resp.Body, NewFileStream(filename))
		f.Sync()
	}
}

type TeeReadCloser struct {
	r io.Reader
	w io.WriteCloser
	c io.Closer
}

func NewTeeReadCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser {
	return &TeeReadCloser{io.TeeReader(r, w), w, r}
}

func (t *TeeReadCloser) Read(b []byte) (int, error) {
	return t.r.Read(b)
}

func (t *TeeReadCloser) Close() error {
	err1 := t.c.Close()
	err2 := t.w.Close()
	if err1 == nil && err2 == nil {<commit_msg>Fix incorrect permissioning on directory creation.<commit_after>package main

import (
	"errors"
	"github.com/elazarl/goproxy"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

type StubResolver struct {
	FileChecker        FileChecker
	UseHostAndProtocol bool
	StubRoot           string
}

func (sr StubResolver) ReturnFileResponse() ResponseGenerator {
	return func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		var resolvedFilename = constructFilename(r.Proto, r.Host, r.URL.Path, r.Method, sr.UseHostAndProtocol, sr.StubRoot)

		var fileContent, _ = ioutil.ReadFile(resolvedFilename)
		fileContentString := string(fileContent[:])
		statusCode := resolveStatusCode(fileContentString)

		fileContentString = stripMetaData(fileContentString)

		return r, goproxy.NewResponse(r, "application/json", statusCode, fileContentString)
	}
}

func (sr StubResolver) CheckFilesystemForRequest() goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		var filename = constructFilename(req.Proto, req.Host, req.URL.Path, req.Method, sr.UseHostAndProtocol, sr.StubRoot)
		if _, err := sr.FileChecker(filename); err == nil {
			return true
		}
		return false
	}
}

type ResponseGenerator func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response)

func constructFilename(proto string, host string, reqPath string, method string, useHostAndProtocol bool, basePath string) string {
	slash := string(os.PathSeparator)
	verb := strings.ToLower(method)
	protocol := strings.ToLower(strings.Split(proto, "/")[0])
	urlpath := reqPath
	if strings.HasSuffix(urlpath, "/") {
		urlpath = urlpath + "index"
	} else {
		urlpath = "/_" + strings.TrimLeft(urlpath, "/")
	}

	hostfilename := path.Join(basePath, path.Clean("."+slash+protocol+slash+host+slash+urlpath+"."+verb+".json"))
	filename := path.Join(basePath, path.Clean("."+slash+urlpath+"."+verb+".json"))

	if useHostAndProtocol {
		return hostfilename
	} else {
		return filename
	}
}

type FileChecker func(name string) (fi os.FileInfo, err error)

func resolveStatusCode(s string) int {
	regex := regexp.MustCompile(`//!.+statusCode:\s+(\d{3})`)
	result := regex.FindStringSubmatch(s)
	if len(result) > 0 {
		intresult, _ := strconv.Atoi(result[1])
		return intresult
	}
	return 200
}

func stripMetaData(s string) string {
	if strings.HasPrefix(s, "//!") {
		stringarray := strings.SplitAfterN(s, "\n", 2)
		return stringarray[1]
	}
	return s
}

type ResponseFunc func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response

func (sr StubResolver) RecordResponse() ResponseFunc {
	return func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		var filename = constructFilename(ctx.Req.Proto, ctx.Req.Host, ctx.Req.URL.Path, ctx.Req.Method, sr.UseHostAndProtocol, sr.StubRoot)
		sr.saveResponse(resp, ctx, filename)
		return resp
	}
}

func (sr StubResolver) saveResponse(resp *http.Response, ctx *goproxy.ProxyCtx, filename string) {
	if resp != nil {
		//TODO Cleanup error
		ctx.Logf("Writing response to here: " + filename)
		if _, err := sr.FileChecker(filepath.Dir(filename)); err != nil {
			os.MkdirAll(filepath.Dir(filename), 0777)
		}
		f, _ := os.Create(filename)
		defer f.Close()

		resp.Body = NewTeeReadCloser(resp.Body, NewFileStream(filename))
		f.Sync()
	}
}

type TeeReadCloser struct {
	r io.Reader
	w io.WriteCloser
	c io.Closer
}

func NewTeeReadCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser {
	return &TeeReadCloser{io.TeeReader(r, w), w, r}
}

func (t *TeeReadCloser) Read(b []byte) (int, error) {
	return t.r.Read(b)
}

func (t *TeeReadCloser) Close() error {
	err1 := t.c.Close()
	err2 := t.w.Close()
	if err1 == nil && err2 == nil {
{'fraction_non_alphanumeric': 0.14128833534217222, 'fraction_numerical': 0.004553368153207446, 'mean_word_length': 4.740199846272098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1252409', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2662, 'n_words': 577}