<commit_before>/**
 * Author:        Tony.Shao
 * Email:         <EMAIL>
 * Github:        github.com/xiocode
 * File:          client.go
 * Description:   httplib client
 */

package httplib

import (
	"io"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sync"
	"time"
)

type status int

type Client struct {
	sync.Mutex
	Conn    *http.Client
	session http.Header
}

func NewClient() *Client {
	return &Client{
		Conn: &http.Client{
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					deadline := time.Now().Add(5 * time.Second)
					conn, err := net.DialTimeout(network, addr, 5*time.Second)
					if err != nil {
						return nil, err
					}
					conn.SetDeadline(deadline)
					return conn, nil
				},
				ResponseHeaderTimeout: 5 * time.Second,
			},
		},
	}
}

func NewProxyClient(proxy string) *Client {
	proxyURL, _ := url.Parse(proxy)
	transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)}
	return &Client{
		Conn: &http.Client{
			Transport: transport,
		},
	}
}

func NewSession() *Client {
	jar, _ := cookiejar.New(nil)
	return &Client{
		Conn: &http.Client{
			Jar: jar,
		},
		session: make(http.Header),
	}
}

func NewProxySession(proxy string) *Client {
	proxyURL, _ := url.Parse(proxy)
	transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)}
	jar, _ := cookiejar.New(nil)
	return &Client{
		Conn: &http.Client{
			Jar:       jar,
			Transport: transport,
		},
		session: make(http.Header),
	}<commit_msg>fix bug<commit_after>/**
 * Author:        Tony.Shao
 * Email:         <EMAIL>
 * Github:        github.com/xiocode
 * File:          client.go
 * Description:   httplib client
 */

package httplib

import (
	"io"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sync"
	"time"
)

type status int

type Client struct {
	sync.Mutex
	Conn    *http.Client
	session http.Header
}

func NewClient() *Client {
	return &Client{
		Conn: &http.Client{
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					deadline := time.Now().Add(5 * time.Second)
					conn, err := net.DialTimeout(network, addr, 5*time.Second)
					if err != nil {
						return nil, err
					}
					conn.SetDeadline(deadline)
					return conn, nil
				},
				ResponseHeaderTimeout: 5 * time.Second,
			},
		},
	}
}

func NewProxyClient(proxy string) *Client {
	proxyURL, _ := url.Parse(proxy)
	return &Client{
		Conn: &http.Client{
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					deadline := time.Now().Add(5 * time.Second)
					conn, err := net.DialTimeout(network, addr, 5*time.Second)
					if err != nil {
						return nil, err
					}
					conn.SetDeadline(deadline)
					return conn, nil
				},
				Proxy: http.ProxyURL(proxyURL),
				ResponseHeaderTimeout: 5 * time.Second,
			},
		},
	}
}

func NewSession() *Client {
	jar, _ := cookiejar.New(nil)
	return &Client{
		Conn: &http.Client{
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					deadline := time.Now().Add(5 * time.Second)
					conn, err := net.DialTimeout(network, addr, 5*time.Second)
					if err != nil {
						return nil, err
					}
					conn.SetDeadline(deadline)
					return conn, nil
				},
				ResponseHeaderTimeout: 5 * time.Second,
			},
			Jar: jar,
		},
		session: make(http.Header),
	}
}

func NewProxySession(proxy string) *Client {
	proxyURL, _ := url.Parse(proxy)
	transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)}
	jar, _ := cookiejar.New(nil)
	return &Client{
		Conn: &http.Client{
			Jar: jar,
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					deadline := time.Now().Add(5 * time.Second)
					conn, err := net.DialTimeout(network, addr, 5*time.Second)
					if err != nil {
						return nil, err
					}
					conn.SetDeadline(deadline)
					return conn, nil
				},
				Proxy: http.ProxyURL(proxyURL),
				ResponseHeaderTimeout: 5 * time.Second,
			},
		},
		session: make(http.Header),
	}
{'fraction_non_alphanumeric': 0.14514514514514515, 'fraction_numerical': 0.0037537537537537537, 'mean_word_length': 2.891918208373905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024293', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1425, 'n_words': 321}