<commit_before>// Copyright 2016, <NAME> <<EMAIL>>. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package goweb

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"godom"
	. "golog"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"crypto/tls"
	"strconv"
	"strings"
)

const (
	HTTP_GET  = "GET"
	HTTP_POST = "POST"
)

const (
	CONTENT_TYPE_DEFAULT     = ""
	CONTENT_TYPE_FORM        = "application/x-www-form-urlencoded"
	CONTENT_TYPE_JSON        = "application/json"
	CONTENT_TYPE_FORM_MULTI  = "multipart/form-data"
)

type HTTP struct {
	Method    string
	URL       *url.URL
	ProxyURL  *url.URL
	cookieJar http.CookieJar
	req       *http.Request
	resp      *http.Response
	Contents  string
}

//
// Constructor
//
func NewHTTP() *HTTP {
	self := &HTTP{}
	self.cookieJar, _ = cookiejar.New(nil)

	return self
}

//
// Tidy the URL such that it is minimally valid
//
func (self *HTTP) _tidyURL(urlString string) (err error) {
	LogDebugf("tidyURL input: %s", urlString)
	url, err := url.Parse(urlString)

	if self.URL == nil {
		if len(url.Scheme) == 0 {
			url.Scheme = "http"
		}

		self.URL = url
	} else {
		if len(url.Scheme) == 0 {
			url.Scheme = self.URL.Scheme
		}

		if len(url.Host) == 0 {
			url.Host = self.URL.Host
		}

		if !strings.HasPrefix(url.Path, "/") {
			components := strings.Split(self.URL.Path, "/")
			maxComponent := len(components)
			path := ""
			for i, component := range components {
				if i < maxComponent-1 {
					if len(component) > 0 {
						path += "/" + component
					}
				}
			}

			path += "/" + url.Path
			url.Path = path
		}

		LogDebugf("tidyURL putput: %s", url)
		self.URL = url
	}

	return err
}

func (self *HTTP) URLString() (urlString string) {
	urlString = fmt.Sprintf("%s", self.URL)

	return urlString
}

//
// Fetch: POST request
//
func (self *HTTP) Post(urlString string, args map[string]string) (result string) {
	content := _formatArgs(args)
	result =  self.PostContent(urlString, CONTENT_TYPE_DEFAULT, content)

	return result
}

func (self *HTTP) PostString(urlString string, contentType string, contentString string) (result string) {
	content := bytes.NewBuffer([]byte(contentString))
	result =  self.PostContent(urlString, contentType, content)

	return result
}

func (self *HTTP) PostData(urlString string, contentType string, contentBytes []byte) (result string) {
	content := bytes.NewBuffer(contentBytes)
	result =  self.PostContent(urlString, contentType, content)

	return result
}

//
// Fetch: POST request
//
func (self *HTTP) PostContent(urlString string, contentType string, content *bytes.Buffer) (result string) {
	self.Method = HTTP_POST
	self._tidyURL(urlString)

	result = self.prepareAndExecuteRequest(contentType, content)
	LogDebugf("\t%d", self.Status())

	return result
}

//
// Fetch: GET request
//
func (self *HTTP) Get(urlString string) (result string) {
	self.Method = HTTP_GET
	self._tidyURL(urlString)

	result = self.prepareAndExecuteRequest(CONTENT_TYPE_DEFAULT, nil)
	LogDebugf("\t%d", self.Status())

	return result
}

//
//
//
func _formatArgs(args map[string]string) (content *bytes.Buffer) {
	if len(args) > 0 {
		argString := ""
		for key, val := range args {
			argString += key + "=" + val + "&"
		}
		content = bytes.NewBuffer([]byte(argString))
	} else {
		content = bytes.NewBuffer([]byte(""))
	}

	return content
}

//
// Fetch: Prepare and execute HTTP request
// NOTE: This is the work horse, all requests filter through here
//
func (self *HTTP) prepareAndExecuteRequest(contentType string, content *bytes.Buffer) string {
	LogDebug(self.Method + ": " + self.URLString())

	RedirectAttemptedError := errors.New("redirect")

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			// return http.ErrUseLastResponse -- Go 1.7+ only
			return RedirectAttemptedError
		},
		Jar: self.cookieJar,
	}

	// if we're proxying, we're going to disable the TLS cert verification
	if self.detectProxy() {
		self.ProxyURL, _ = url.Parse("http://127.0.0.1:8080")
		client.Transport = &http.Transport{
			Proxy: http.ProxyURL(self.ProxyURL),
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	self.req, _ = http.NewRequest(self.Method, self.URLString(), content)

	if len(contentType) > 0 {
		self.req.Header.Add("Content-Type", contentType)
	}
	self.req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50")
	// self.req.Header.Add("Referer", referrer)
	var err error
	self.resp, err = client.Do(self.req)

	if err != nil && !strings.HasSuffix(err.Error(), " redirect") {
		LogFatal(err)
	} else {
		bytes, _ := ioutil.ReadAll(self.resp.Body)
		self.Contents = string(bytes)<commit_msg>nils are bad, mmmmkay<commit_after>// Copyright 2016, <NAME> <<EMAIL>>. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package goweb

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"godom"
	. "golog"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"crypto/tls"
	"strconv"
	"strings"
)

const (
	HTTP_GET  = "GET"
	HTTP_POST = "POST"
)

const (
	CONTENT_TYPE_DEFAULT     = ""
	CONTENT_TYPE_FORM        = "application/x-www-form-urlencoded"
	CONTENT_TYPE_JSON        = "application/json"
	CONTENT_TYPE_FORM_MULTI  = "multipart/form-data"
)

type HTTP struct {
	Method    string
	URL       *url.URL
	ProxyURL  *url.URL
	cookieJar http.CookieJar
	req       *http.Request
	resp      *http.Response
	Contents  string
}

//
// Constructor
//
func NewHTTP() *HTTP {
	self := &HTTP{}
	self.cookieJar, _ = cookiejar.New(nil)

	return self
}

//
// Tidy the URL such that it is minimally valid
//
func (self *HTTP) _tidyURL(urlString string) (err error) {
	LogDebugf("tidyURL input: %s", urlString)
	url, err := url.Parse(urlString)

	if self.URL == nil {
		if len(url.Scheme) == 0 {
			url.Scheme = "http"
		}

		self.URL = url
	} else {
		if len(url.Scheme) == 0 {
			url.Scheme = self.URL.Scheme
		}

		if len(url.Host) == 0 {
			url.Host = self.URL.Host
		}

		if !strings.HasPrefix(url.Path, "/") {
			components := strings.Split(self.URL.Path, "/")
			maxComponent := len(components)
			path := ""
			for i, component := range components {
				if i < maxComponent-1 {
					if len(component) > 0 {
						path += "/" + component
					}
				}
			}

			path += "/" + url.Path
			url.Path = path
		}

		LogDebugf("tidyURL putput: %s", url)
		self.URL = url
	}

	return err
}

func (self *HTTP) URLString() (urlString string) {
	urlString = fmt.Sprintf("%s", self.URL)

	return urlString
}

//
// Fetch: POST request
//
func (self *HTTP) Post(urlString string, args map[string]string) (result string) {
	content := _formatArgs(args)
	result =  self.PostContent(urlString, CONTENT_TYPE_DEFAULT, content)

	return result
}

func (self *HTTP) PostString(urlString string, contentType string, contentString string) (result string) {
	content := bytes.NewBuffer([]byte(contentString))
	result =  self.PostContent(urlString, contentType, content)

	return result
}

func (self *HTTP) PostData(urlString string, contentType string, contentBytes []byte) (result string) {
	content := bytes.NewBuffer(contentBytes)
	result =  self.PostContent(urlString, contentType, content)

	return result
}

//
// Fetch: POST request
//
func (self *HTTP) PostContent(urlString string, contentType string, content *bytes.Buffer) (result string) {
	self.Method = HTTP_POST
	self._tidyURL(urlString)

	result = self.prepareAndExecuteRequest(contentType, content)
	LogDebugf("\t%d", self.Status())

	return result
}

//
// Fetch: GET request
//
func (self *HTTP) Get(urlString string) (result string) {
	self.Method = HTTP_GET
	self._tidyURL(urlString)

	result = self.prepareAndExecuteRequest(CONTENT_TYPE_DEFAULT, nil)
	LogDebugf("\t%d", self.Status())

	return result
}

//
//
//
func _formatArgs(args map[string]string) (content *bytes.Buffer) {
	if len(args) > 0 {
		argString := ""
		for key, val := range args {
			argString += key + "=" + val + "&"
		}
		content = bytes.NewBuffer([]byte(argString))
	} else {
		content = bytes.NewBuffer([]byte(""))
	}

	return content
}

//
// Fetch: Prepare and execute HTTP request
// NOTE: This is the work horse, all requests filter through here
//
func (self *HTTP) prepareAndExecuteRequest(contentType string, content *bytes.Buffer) string {
	LogDebug(self.Method + ": " + self.URLString())

	RedirectAttemptedError := errors.New("redirect")

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			// return http.ErrUseLastResponse -- Go 1.7+ only
			return RedirectAttemptedError
		},
		Jar: self.cookieJar,
	}

	// if we're proxying, we're going to disable the TLS cert verification
	if self.detectProxy() {
		self.ProxyURL, _ = url.Parse("http://127.0.0.1:8080")
		client.Transport = &http.Transport{
			Proxy: http.ProxyURL(self.ProxyURL),
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	// ensure we don't pass a nil ByteBuffer into http.NewRequest
	if content == nil {
		content = bytes.NewBuffer([]byte(""))
	}

	self.req, _ = http.NewRequest(self.Method, self.URLString(), content)

	if len(contentType) > 0 {
		self.req.Header.Add("Content-Type", contentType)
	}
	self.req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50")
	// self.req.Header.Add("Referer", referrer)
	var err error
	self.resp, err = client.Do(self.req)

	if err != nil && !strings.HasSuffix(err.Error(), " redirect") {
		LogFatal(err)
	} else {
		bytes, _ := ioutil.ReadAll(self.resp.Body)
		self.Contents = string(bytes)
{'fraction_non_alphanumeric': 0.14546753777051857, 'fraction_numerical': 0.00898325847284606, 'mean_word_length': 3.6234072675790467, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4676629', 'n_tokens_mistral': 3719, 'n_tokens_neox': 3438, 'n_words': 830}