<commit_before>package net

import (
	"encoding/json"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"net/http"
	"strings"
)

type JSONMapRequest map[string]interface{}

func (json *JSONMapRequest) String() string {
	return fmt.Sprintf("%#v", *json)
}

type JSONArrayRequest []interface{}

func (json *JSONArrayRequest) String() string {
	return fmt.Sprintf("%#v", *json)
}

func bytesToInterface(jsonBytes []byte) (interface{}, error) {
	mapResult := &JSONMapRequest{}
	err := json.Unmarshal(jsonBytes, mapResult)
	if err == nil {
		return mapResult, err
	}

	arrayResult := &JSONArrayRequest{}
	err = json.Unmarshal(jsonBytes, arrayResult)
	return arrayResult, err
}

func RequestBodyMatcher(expectedBodyString string) RequestMatcher {
	return func(request *http.Request) {
		bodyBytes, err := ioutil.ReadAll(request.Body)
		if err != nil {
			Fail(fmt.Sprintf("Error reading request body: %s", err))
		}

		actualBody, err := bytesToInterface(bodyBytes)
		if err != nil {
			Fail(fmt.Sprintf("Error unmarshalling request", err.Error()))
		}

		expectedBody, err := bytesToInterface([]byte(expectedBodyString))
		if err != nil {
			Fail(fmt.Sprintf("Error unmarshalling expected json", err.Error()))
		}

		Expect(expectedBody).To(Equal(actualBody), "\nEXPECTED: %s\nACTUAL: %s", expectedBody, actualBody)
		Expect(request.Header.Get("content-type")).To(Equal("application/json"), "Content Type was not application/json.")
	}
}

func RequestBodyMatcherWithContentType(expectedBody, expectedContentType string) RequestMatcher {
	return func(request *http.Request) {
		bodyBytes, err := ioutil.ReadAll(request.Body)
		if err != nil {
			Fail(fmt.Sprintf("Error reading request body: %s", err))
		}

		actualBody := string(bodyBytes)
		Expect(RemoveWhiteSpaceFromBody(actualBody)).To(Equal(RemoveWhiteSpaceFromBody(expectedBody)), "Body did not match.")

		actualContentType := request.Header.Get("content-type")
		Expect(actualContentType).To(Equal(expectedContentType), "Content Type did not match.")
	}
}

func RemoveWhiteSpaceFromBody(body string) string {
	body = strings.Replace(body, " ", "", -1)
	body = strings.Replace(body, "\n", "", -1)
	body = strings.Replace(body, "\r", "", -1)
	body = strings.Replace(body, "\t", "", -1)
	return body
}<commit_msg>GinkgoRecover() in goroutines<commit_after>package net

import (
	"encoding/json"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"net/http"
	"strings"
)

type JSONMapRequest map[string]interface{}

func (json *JSONMapRequest) String() string {
	return fmt.Sprintf("%#v", *json)
}

type JSONArrayRequest []interface{}

func (json *JSONArrayRequest) String() string {
	return fmt.Sprintf("%#v", *json)
}

func bytesToInterface(jsonBytes []byte) (interface{}, error) {
	mapResult := &JSONMapRequest{}
	err := json.Unmarshal(jsonBytes, mapResult)
	if err == nil {
		return mapResult, err
	}

	arrayResult := &JSONArrayRequest{}
	err = json.Unmarshal(jsonBytes, arrayResult)
	return arrayResult, err
}

func RequestBodyMatcher(expectedBodyString string) RequestMatcher {
	return func(request *http.Request) {
		defer GinkgoRecover()
		bodyBytes, err := ioutil.ReadAll(request.Body)
		if err != nil {
			Fail(fmt.Sprintf("Error reading request body: %s", err))
		}

		actualBody, err := bytesToInterface(bodyBytes)
		if err != nil {
			Fail(fmt.Sprintf("Error unmarshalling request", err.Error()))
		}

		expectedBody, err := bytesToInterface([]byte(expectedBodyString))
		if err != nil {
			Fail(fmt.Sprintf("Error unmarshalling expected json", err.Error()))
		}

		Expect(expectedBody).To(Equal(actualBody), "\nEXPECTED: %s\nACTUAL: %s", expectedBody, actualBody)
		Expect(request.Header.Get("content-type")).To(Equal("application/json"), "Content Type was not application/json.")
	}
}

func RequestBodyMatcherWithContentType(expectedBody, expectedContentType string) RequestMatcher {
	return func(request *http.Request) {
		defer GinkgoRecover()
		bodyBytes, err := ioutil.ReadAll(request.Body)
		if err != nil {
			Fail(fmt.Sprintf("Error reading request body: %s", err))
		}

		actualBody := string(bodyBytes)
		Expect(RemoveWhiteSpaceFromBody(actualBody)).To(Equal(RemoveWhiteSpaceFromBody(expectedBody)), "Body did not match.")

		actualContentType := request.Header.Get("content-type")
		Expect(actualContentType).To(Equal(expectedContentType), "Content Type did not match.")
	}
}

func RemoveWhiteSpaceFromBody(body string) string {
	body = strings.Replace(body, " ", "", -1)
	body = strings.Replace(body, "\n", "", -1)
	body = strings.Replace(body, "\r", "", -1)
	body = strings.Replace(body, "\t", "", -1)
	return body
}
{'fraction_non_alphanumeric': 0.15152819629789066, 'fraction_numerical': 0.0017219113215669393, 'mean_word_length': 5.305291723202171, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2257474', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1496, 'n_words': 334}