<commit_before>package steps

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"github.com/stretchr/testify/assert"
)

// registers all HTTP-check related steps
func init() {
	RegisterSteps(func(c *Context) {

		c.Then(`^I should get an HTTP response code (\d+) on path "(.+?)" through the tunnel "(.+?)"$`, func(expectedResponseCode int, path string, tunnelName string) {
			tunnel := c.GetTunnel(tunnelName)
			if tunnel == nil {
				c.Fail("Could not find a tunnel named '%s'", tunnelName)
				return
			}

			url := fmt.Sprintf("http://%s:%v%s", "localhost", tunnel.LocalPort, path)
			resp, err := c.execHttpGetRequest(url)
			if err != nil {
				c.Fail("HTTP request on %s failed: %v", url, err)
				return
			}

			assert.Equal(c.T, expectedResponseCode, resp.StatusCode)
			resp.Body.Close()
		})

		c.Then(`^I should have the text "(.+?)" on path "(.+?)" through the tunnel "(.+?)"$`, func(expectedContentText string, path string, tunnelName string) {
			tunnel := c.GetTunnel(tunnelName)
			if tunnel == nil {
				c.Fail("Could not find a tunnel named '%s'", tunnelName)
				return
			}

			url := fmt.Sprintf("http://%s:%v%s", "localhost", tunnel.LocalPort, path)
			resp, err := c.execHttpGetRequest(url)
			if err != nil {
				c.Fail("HTTP request on %s failed: %v", url, err)
				return
			}
			data, err := ioutil.ReadAll(resp.Body)

			assert.Contains(c.T, string(data), expectedContentText)
			resp.Body.Close()
		})

	})
}

// execHttpGetRequest executes an HTTP GET request on the given URL
// and returns the response or an error
// It uses an exponential backoff retry
func (c *Context) execHttpGetRequest(url string) (*http.Response, error) {
	transport := &http.Transport{
		DisableKeepAlives:     true,
		MaxIdleConnsPerHost:   5,
		ResponseHeaderTimeout: 10 * time.Second,
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
	}
	client := &http.Client{
		Transport: transport,
		Timeout:   5 * time.Second,
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	var resp *http.Response
	err = c.ExecWithExponentialBackoff(func() error {
		var err error
		resp, err = client.Do(req)
		return err
	})
<commit_msg>add http response header checks<commit_after>package steps

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/stretchr/testify/assert"
)

// registers all HTTP-check related steps
func init() {
	var currentResp *http.Response
	requestHeaders := make(http.Header)

	RegisterSteps(func(c *Context) {

		c.Then(`^I should get an HTTP response code (\d+) on path "(.+?)" through the tunnel "(.+?)"$`, func(expectedResponseCode int, path string, tunnelName string) {
			tunnel := c.GetTunnel(tunnelName)
			if tunnel == nil {
				c.Fail("Could not find a tunnel named '%s'", tunnelName)
				return
			}

			url := fmt.Sprintf("http://%s:%v%s", "localhost", tunnel.LocalPort, path)
			resp, err := c.execHttpGetRequest(url, requestHeaders)
			currentResp = resp
			if err != nil {
				c.Fail("HTTP request on %s failed: %v", url, err)
				return
			}

			assert.Equal(c.T, expectedResponseCode, resp.StatusCode)
			resp.Body.Close()
		})

		c.Then(`^This response has header "(.+?)" equals to "(.+?)"$`, func(name, expectedValue string) {
			assert.Equal(c.T, expectedValue, currentResp.Header.Get(name))
		})

		c.Then(`^This response has no header "(.+?)"$`, func(name string) {
			assert.NotContains(c.T, name, currentResp.Header)
		})

		c.Then(`^I should have the text "(.+?)" on path "(.+?)" through the tunnel "(.+?)"$`, func(expectedContentText string, path string, tunnelName string) {
			tunnel := c.GetTunnel(tunnelName)
			if tunnel == nil {
				c.Fail("Could not find a tunnel named '%s'", tunnelName)
				return
			}

			url := fmt.Sprintf("http://%s:%v%s", "localhost", tunnel.LocalPort, path)
			resp, err := c.execHttpGetRequest(url, requestHeaders)
			if err != nil {
				c.Fail("HTTP request on %s failed: %v", url, err)
				return
			}
			data, err := ioutil.ReadAll(resp.Body)

			assert.Contains(c.T, string(data), expectedContentText)
			resp.Body.Close()
		})

		c.Given(`^Using request header "(.+?)": "(.+?)"$`, func(name, value string) {
			for _, v := range strings.Split(value, ";") {
				requestHeaders.Add(name, v)
			}
		})

	})
}

// execHttpGetRequest executes an HTTP GET request on the given URL
// and returns the response or an error
// It uses an exponential backoff retry
func (c *Context) execHttpGetRequest(url string, headers http.Header) (*http.Response, error) {
	transport := &http.Transport{
		DisableKeepAlives:     true,
		MaxIdleConnsPerHost:   5,
		ResponseHeaderTimeout: 10 * time.Second,
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
	}
	client := &http.Client{
		Transport: transport,
		Timeout:   5 * time.Second,
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	for name, values := range headers {
		for _, value := range values {
			req.Header.Add(name, value)
		}
	}

	var resp *http.Response
	err = c.ExecWithExponentialBackoff(func() error {
		var err error
		resp, err = client.Do(req)
		return err
	})

{'fraction_non_alphanumeric': 0.15741281901422172, 'fraction_numerical': 0.0019481784531463083, 'mean_word_length': 3.5595026642984013, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983967', 'n_tokens_mistral': 1980, 'n_tokens_neox': 1752, 'n_words': 479}