<commit_before>package task

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"neon/build"
	"neon/util"
	"net/http"
	"strconv"
)

const (
	DEFAULT_METHOD = "GET"
	DEFAULT_STATUS = "200"
)

func init() {
	build.TaskMap["request"] = build.TaskDescriptor{
		Constructor: Request,
		Help: `Perform an HTTP request.

Arguments:

- method: the request method (GET, POST, etc), defaults to "GET".
- headers: request headers as a map
- body: the request body as a string.
- json: the request body to send as Json.
- file: the request body as a file.
- status: expected status code, on error if different (defaults to 200).
- username: user name for authentication.
- password: <PASSWORD>.

Response status code is stored in variable _status, response body is stored in
variable _body and response headers in _headers.

Examples:

    # get google.com
    - request: "google.com"`,
	}
}

func Request(target *build.Target, args util.Object) (build.Task, error) {
	fields := []string{"request", "method", "headers", "body", "json", "file", "status", "username", "password"}
	if err := CheckFields(args, fields, fields[:1]); err != nil {
		return nil, err
	}
	url, err := args.GetString("request")
	if err != nil {
		return nil, fmt.Errorf("argument request must be a string")
	}
	var method = DEFAULT_METHOD
	if args.HasField("method") {
		method, err = args.GetString("method")
		if err != nil {
			return nil, fmt.Errorf("argument method of task request must be a string")
		}
	}
	var headers map[string]string
	if args.HasField("headers") {
		headers, err = args.GetMapStringString("headers")
		if err != nil {
			return nil, fmt.Errorf("argument headers of task request must be a map of strings strings")
		}
	}
	var body string
	if args.HasField("body") {
		body, err = args.GetString("body")
		if err != nil {
			return nil, fmt.Errorf("argument body of task request must be a string")
		}
	}
	var json interface{}
	if args.HasField("json") {
		json = args["json"]
	}
	var file string
	if args.HasField("file") {
		file, err = args.GetString("file")
		if err != nil {
			return nil, fmt.Errorf("argument file of task request must be a string")
		}
	}
	var status string
	if args.HasField("status") {
		status, err = args.GetString("status")
		if err != nil {
			return nil, fmt.Errorf("argument status of task request must be a string")
		}
	}
	var username string
	if args.HasField("username") {
		username, err = args.GetString("username")
		if err != nil {
			return nil, fmt.Errorf("argument username of task request must be a string")
		}
	}
	var password string
	if args.HasField("password") {
		password, err = args.GetString("password")
		if err != nil {
			return nil, fmt.Errorf("argument password of task request must be a string")
		}
	}
	if file != "" && body != "" {
		return nil, fmt.Errorf("body and file can't be set at the same time")
	}
	return func() error {
		// evaluate arguments
		_url, _err := target.Build.Context.EvaluateString(url)
		if _err != nil {
			return fmt.Errorf("evaluating url: %v", _err)
		}
		_method, _err := target.Build.Context.EvaluateString(method)
		if _err != nil {
			return fmt.Errorf("evaluating method: %v", _err)
		}
		_headers := make(map[string]string)
		for name, value := range headers {
			_name, _err := target.Build.Context.EvaluateString(name)
			if _err != nil {
				return fmt.Errorf("evaluating headers: %v", _err)
			}
			_value, _err := target.Build.Context.EvaluateString(value)
			if _err != nil {
				return fmt.Errorf("evaluating headers: %v", _err)
			}
			_headers[_name] = _value
		}
		_body, _err := target.Build.Context.EvaluateString(body)
		if _err != nil {
			return fmt.Errorf("evaluating body: %v", _err)
		}
		var _json = ""
		if json != nil {
			_json, _err = build.PropertyToString(json, true)
			if err != nil {
				return fmt.Errorf("serializing json argument: %v", err)
			}
		}
		_file, _err := target.Build.Context.EvaluateString(file)
		if _err != nil {
			return fmt.Errorf("evaluating file: %v", _err)
		}
		if _file != "" {
			_file = util.ExpandAndJoinToRoot(target.Build.Dir, _file)
		}
		_status, _err := target.Build.Context.EvaluateString(status)
		if _err != nil {
			return fmt.Errorf("evaluating status: %v", _err)
		}
		if _status == "" {
			_status = DEFAULT_STATUS
		}
		_username, _err := target.Build.Context.EvaluateString(username)
		if _err != nil {
			return fmt.Errorf("evaluating username: %v", _err)
		}
		_password, _err := target.Build.Context.EvaluateString(password)
		if _err != nil {
			return fmt.Errorf("evaluating password: %v", _err)
		}
		// perform request
		if _json != "" {
			_body = _json
		}<commit_msg>Removed json field (use body)<commit_after>package task

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"neon/build"
	"neon/util"
	"net/http"
	"strconv"
)

const (
	DEFAULT_METHOD = "GET"
	DEFAULT_STATUS = "200"
)

func init() {
	build.TaskMap["request"] = build.TaskDescriptor{
		Constructor: Request,
		Help: `Perform an HTTP request.

Arguments:

- method: the request method (GET, POST, etc), defaults to "GET".
- headers: request headers as a map
- body: the request body as a string.
- file: the request body as a file.
- status: expected status code, on error if different (defaults to 200).
- username: user name for authentication.
- password: <PASSWORD>.

Response status code is stored in variable _status, response body is stored in
variable _body and response headers in _headers.

Examples:

    # get google.com
    - request: "google.com"`,
	}
}

func Request(target *build.Target, args util.Object) (build.Task, error) {
	fields := []string{"request", "method", "headers", "body", "file", "status", "username", "password"}
	if err := CheckFields(args, fields, fields[:1]); err != nil {
		return nil, err
	}
	url, err := args.GetString("request")
	if err != nil {
		return nil, fmt.Errorf("argument request must be a string")
	}
	var method = DEFAULT_METHOD
	if args.HasField("method") {
		method, err = args.GetString("method")
		if err != nil {
			return nil, fmt.Errorf("argument method of task request must be a string")
		}
	}
	var headers map[string]string
	if args.HasField("headers") {
		headers, err = args.GetMapStringString("headers")
		if err != nil {
			return nil, fmt.Errorf("argument headers of task request must be a map of strings strings")
		}
	}
	var body string
	if args.HasField("body") {
		body, err = args.GetString("body")
		if err != nil {
			return nil, fmt.Errorf("argument body of task request must be a string")
		}
	}
	var file string
	if args.HasField("file") {
		file, err = args.GetString("file")
		if err != nil {
			return nil, fmt.Errorf("argument file of task request must be a string")
		}
	}
	var status string
	if args.HasField("status") {
		status, err = args.GetString("status")
		if err != nil {
			return nil, fmt.Errorf("argument status of task request must be a string")
		}
	}
	var username string
	if args.HasField("username") {
		username, err = args.GetString("username")
		if err != nil {
			return nil, fmt.Errorf("argument username of task request must be a string")
		}
	}
	var password string
	if args.HasField("password") {
		password, err = args.GetString("password")
		if err != nil {
			return nil, fmt.Errorf("argument password of task request must be a string")
		}
	}
	if file != "" && body != "" {
		return nil, fmt.Errorf("body and file can't be set at the same time")
	}
	return func() error {
		// evaluate arguments
		_url, _err := target.Build.Context.EvaluateString(url)
		if _err != nil {
			return fmt.Errorf("evaluating url: %v", _err)
		}
		_method, _err := target.Build.Context.EvaluateString(method)
		if _err != nil {
			return fmt.Errorf("evaluating method: %v", _err)
		}
		_headers := make(map[string]string)
		for name, value := range headers {
			_name, _err := target.Build.Context.EvaluateString(name)
			if _err != nil {
				return fmt.Errorf("evaluating headers: %v", _err)
			}
			_value, _err := target.Build.Context.EvaluateString(value)
			if _err != nil {
				return fmt.Errorf("evaluating headers: %v", _err)
			}
			_headers[_name] = _value
		}
		_body, _err := target.Build.Context.EvaluateString(body)
		if _err != nil {
			return fmt.Errorf("evaluating body: %v", _err)
		}
		_file, _err := target.Build.Context.EvaluateString(file)
		if _err != nil {
			return fmt.Errorf("evaluating file: %v", _err)
		}
		if _file != "" {
			_file = util.ExpandAndJoinToRoot(target.Build.Dir, _file)
		}
		_status, _err := target.Build.Context.EvaluateString(status)
		if _err != nil {
			return fmt.Errorf("evaluating status: %v", _err)
		}
		if _status == "" {
			_status = DEFAULT_STATUS
		}
		_username, _err := target.Build.Context.EvaluateString(username)
		if _err != nil {
			return fmt.Errorf("evaluating username: %v", _err)
		}
		_password, _err := target.Build.Context.EvaluateString(password)
		if _err != nil {
			return fmt.Errorf("evaluating password: %v", _err)
		}
		// perform request
{'fraction_non_alphanumeric': 0.13578722926719786, 'fraction_numerical': 0.0015710919088766694, 'mean_word_length': 3.65378590078329, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066617', 'n_tokens_mistral': 3175, 'n_tokens_neox': 2915, 'n_words': 914}