<commit_before>// Copyright 2014 <NAME>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ht

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/vdobler/ht/check"
	"github.com/vdobler/ht/response"
)

var (
	// DefaultUserAgent is the user agent string to send in http requests
	// if no user agent header is specified explicitely.
	DefaultUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36"

	// DefaultAccept is the accept header to be sent if no accept header
	// is set explicitely in the test.
	DefaultAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

	// DefaultClientTimeout is the timeout used by the http clients.
	DefaultClientTimeout = 2 * time.Second
)

// Request is a HTTP request.
type Request struct {
	// Method is the HTTP method to use.
	// A empty method is equivalent to "GET"
	Method string `json:",omitempty"`

	// URL ist the URL of the request.
	URL string

	// Params contains the parameters and their values to send in
	// the request.
	//
	// If the parameters are sent as multipart it is possible to include
	// files: If a parameter values starts with "@file:" the rest of
	// the value is interpreted as as filename and this file is sent.
	Params url.Values `json:",omitempty"`

	// ParamsAs determines how the parameters in the Param field are sent:
	//   "URL" or "": append properly encoded to URL
	//   "body"     : send as application/x-www-form-urlencoded in body.
	//   "multipart": send as multipart in body.
	// The two values "body" and "multipart" must not be used
	// on a GET or HEAD request.
	ParamsAs string `json:",omitempty"`

	// Header contains the specific http headers to be sent in this request.
	// User-Agent and Accept headers are set automaticaly to the global
	// default values if not set explicitely.
	Header http.Header `json:",omitempty"`

	// Cookies contains the cookies to send in the request.
	Cookies []Cookie `json:",omitempty"`

	// Body is the full body to send in the request. Body must be
	// empty if Params are sent as multipart or form-urlencoded.
	Body string `json:",omitempty"`

	// FollowRedirects determines if automatic following of
	// redirects should be done.
	FollowRedirects bool `json:",omitempty"`
}

// Cookie.
type Cookie struct {
	Name  string
	Value string `json:",omitempty"`
}

// ----------------------------------------------------------------------------
// Test

// Test is a single logical test which does one HTTP request and checks
// a number of Checks on the recieved Response.
type Test struct {
	Name        string
	Description string

	// Request is the HTTP request.
	Request Request

	// Checks contains all checks to perform on the response to the HTTP request.
	Checks check.CheckList

	Poll      Poll          `json:",omitempty"`
	Timeout   time.Duration // If zero use DefaultClientTimeout.
	Verbosity int           // Verbosity level in logging.

	Jar http.CookieJar `json:"-"` // The possible prepopulated cookie jar to use.

	client   *http.Client
	request  *http.Request
	response *response.Response
	checks   []check.Check // compiled checks.
}

// Poll determines if and how to redo a test after a failure or if the
// test should be skipped alltogether. The zero value of Poll means "Just do
// the test once."
type Poll struct {
	// Maximum number of redos. Both 0 and 1 mean: "Just one try. No redo."
	// Negative values indicate that the test should be skipped.
	Max int `json:",omitempty"`

	// Duration to sleep between redos.
	Sleep time.Duration `json:",omitempty"`
}

// Skip return whether the test should be skipped.
func (p Poll) Skip() bool {
	return p.Max < 0
}

func (t *Test) errorf(format string, v ...interface{}) {
	if t.Verbosity >= 0 {
		format = "ERROR " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) infof(format string, v ...interface{}) {
	if t.Verbosity >= 1 {
		format = "INFO " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) debugf(format string, v ...interface{}) {
	if t.Verbosity >= 2 {
		format = "DEBUG " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) tracef(format string, v ...interface{}) {
	if t.Verbosity >= 3 {
		format = "TRACE " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

// Run runs the test t. The actual HTTP request is crafted and executed and
// the checks are performed on the received response. This whole process
// is repeated on failure or skipped entirely according to t.Poll.
//
// Normally all checks in t.Checks are executed. If the first check in
// t.Checks is a StatusCode check against 200 and it fails, then the rest of
// the tests are skipped.
func (t *Test) Run(variables map[string]string) TestResult {
	// Set up a TestResult, prefill static information
	result := TestResult{
		Name:        t.Name,
		Description: t.Description,
	}
	result.CheckResults = make([]CheckResult, len(t.Checks)) // Zero value is NotRun
	for i, c := range t.Checks {
		result.CheckResults[i].Name = check.NameOf(c)
		buf, err := json.Marshal(c)
		if err != nil {
			buf = []byte(err.Error())
		}
		result.CheckResults[i].JSON = string(buf)
	}

	maxTries := t.Poll.Max
	if maxTries == 0 {
		maxTries = 1
	}
	if maxTries < 0 {
		// This test is deliberately skipped.
		result.Status = Skipped
		// TODO: Should Checks be filled with skipped CheckResult?
		return result
	}

	// Try until first success.
	try := 0<commit_msg>Record start time of a test.<commit_after>// Copyright 2014 <NAME>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ht

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/vdobler/ht/check"
	"github.com/vdobler/ht/response"
)

var (
	// DefaultUserAgent is the user agent string to send in http requests
	// if no user agent header is specified explicitely.
	DefaultUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36"

	// DefaultAccept is the accept header to be sent if no accept header
	// is set explicitely in the test.
	DefaultAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

	// DefaultClientTimeout is the timeout used by the http clients.
	DefaultClientTimeout = 2 * time.Second
)

// Request is a HTTP request.
type Request struct {
	// Method is the HTTP method to use.
	// A empty method is equivalent to "GET"
	Method string `json:",omitempty"`

	// URL ist the URL of the request.
	URL string

	// Params contains the parameters and their values to send in
	// the request.
	//
	// If the parameters are sent as multipart it is possible to include
	// files: If a parameter values starts with "@file:" the rest of
	// the value is interpreted as as filename and this file is sent.
	Params url.Values `json:",omitempty"`

	// ParamsAs determines how the parameters in the Param field are sent:
	//   "URL" or "": append properly encoded to URL
	//   "body"     : send as application/x-www-form-urlencoded in body.
	//   "multipart": send as multipart in body.
	// The two values "body" and "multipart" must not be used
	// on a GET or HEAD request.
	ParamsAs string `json:",omitempty"`

	// Header contains the specific http headers to be sent in this request.
	// User-Agent and Accept headers are set automaticaly to the global
	// default values if not set explicitely.
	Header http.Header `json:",omitempty"`

	// Cookies contains the cookies to send in the request.
	Cookies []Cookie `json:",omitempty"`

	// Body is the full body to send in the request. Body must be
	// empty if Params are sent as multipart or form-urlencoded.
	Body string `json:",omitempty"`

	// FollowRedirects determines if automatic following of
	// redirects should be done.
	FollowRedirects bool `json:",omitempty"`
}

// Cookie.
type Cookie struct {
	Name  string
	Value string `json:",omitempty"`
}

// ----------------------------------------------------------------------------
// Test

// Test is a single logical test which does one HTTP request and checks
// a number of Checks on the recieved Response.
type Test struct {
	Name        string
	Description string

	// Request is the HTTP request.
	Request Request

	// Checks contains all checks to perform on the response to the HTTP request.
	Checks check.CheckList

	Poll      Poll          `json:",omitempty"`
	Timeout   time.Duration // If zero use DefaultClientTimeout.
	Verbosity int           // Verbosity level in logging.

	Jar http.CookieJar `json:"-"` // The possible prepopulated cookie jar to use.

	client   *http.Client
	request  *http.Request
	response *response.Response
	checks   []check.Check // compiled checks.
}

// Poll determines if and how to redo a test after a failure or if the
// test should be skipped alltogether. The zero value of Poll means "Just do
// the test once."
type Poll struct {
	// Maximum number of redos. Both 0 and 1 mean: "Just one try. No redo."
	// Negative values indicate that the test should be skipped.
	Max int `json:",omitempty"`

	// Duration to sleep between redos.
	Sleep time.Duration `json:",omitempty"`
}

// Skip return whether the test should be skipped.
func (p Poll) Skip() bool {
	return p.Max < 0
}

func (t *Test) errorf(format string, v ...interface{}) {
	if t.Verbosity >= 0 {
		format = "ERROR " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) infof(format string, v ...interface{}) {
	if t.Verbosity >= 1 {
		format = "INFO " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) debugf(format string, v ...interface{}) {
	if t.Verbosity >= 2 {
		format = "DEBUG " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

func (t *Test) tracef(format string, v ...interface{}) {
	if t.Verbosity >= 3 {
		format = "TRACE " + format + " [%q]"
		v = append(v, t.Name)
		log.Printf(format, v...)
	}
}

// Run runs the test t. The actual HTTP request is crafted and executed and
// the checks are performed on the received response. This whole process
// is repeated on failure or skipped entirely according to t.Poll.
//
// Normally all checks in t.Checks are executed. If the first check in
// t.Checks is a StatusCode check against 200 and it fails, then the rest of
// the tests are skipped.
func (t *Test) Run(variables map[string]string) TestResult {
	// Set up a TestResult, prefill static information
	result := TestResult{
		Name:        t.Name,
		Description: t.Description,
		Started:     time.Now(),
	}
	result.CheckResults = make([]CheckResult, len(t.Checks)) // Zero value is NotRun
	for i, c := range t.Checks {
		result.CheckResults[i].Name = check.NameOf(c)
		buf, err := json.Marshal(c)
		if err != nil {
			buf = []byte(err.Error())
		}
		result.CheckResults[i].JSON = string(buf)
	}

	maxTries := t.Poll.Max
	if maxTries == 0 {
		maxTries = 1
	}
	if maxTries < 0 {
		// This test is deliberately skipped.
		result.Status = Skipped
		// TODO: Should Checks be filled with skipped CheckResult?
		return result
	}

	// Try until first success.
	try := 0
{'fraction_non_alphanumeric': 0.12732598208132323, 'fraction_numerical': 0.008442453480358374, 'mean_word_length': 2.9540190735694822, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7157152', 'n_tokens_mistral': 3801, 'n_tokens_neox': 3597, 'n_words': 1448}