<commit_before>local socket = require "socket"
local ssl = require "ssl"
local stream = require "http.stream"
local dns = require "dns"

local client = {}
local EMPTY = {}

local function parseurl(url)
	local default = false
	local scheme, host, port, path= string.match(url, "(http[s]-)://([^:/]+):?(%d*)([%w-%.?&=_/]*)")
	if path == "" then
		path = "/"
	end
	if port == "" then
		if scheme == "https" then
			port = "443"
		elseif scheme == "http" then
			port = "80"
		else
			assert(false, "unsupport parse url scheme:" .. scheme)
		end
		default = true
	end
	return scheme, host, port, path, default
end

local function send_request(io_do, fd, method, host, abs, header, body)
	local tmp = ""
	table.insert(header, 1, string.format("%s %s HTTP/1.1", method, abs))
	table.insert(header, string.format("Host: %s", host))
	table.insert(header, string.format("Content-Length: %d", #body))
	table.insert(header, "User-Agent: Silly/0.2")
	table.insert(header, "Connection: keep-alive")
	table.insert(header, "\r\n")
	tmp = table.concat(header, "\r\n")
	tmp = tmp .. body
	io_do.write(fd, tmp)
end

local function recv_response(io_do, fd)
	local readl = function()
		return io_do.readline(fd, "\r\n")
	end
	local readn = function(n)
		return io_do.read(fd, n)
	end
	local status, first, header, body = stream.recv_request(readl, readn)
	if not status then	--disconnected
		return nil
	end
	if status ~= 200 then
		return status
	end
	local ver, status= first:match("HTTP/([%d|.]+)%s+(%d+)")
	return tonumber(status), header, body, ver
end

local function process(uri, method, header, body)
	local ip, io_do
	local scheme, host, port, path, default = parseurl(uri)
	if dns.isdomain(host) then
		ip = dns.query(host)
	else
		ip = host
	end
	if scheme == "https" then
		io_do = ssl
	elseif scheme == "http" then
		io_do = socket
	end
	if not default then
		host = string.format("%s:%s", host, port)
	end
	ip = string.format("%s@%s", ip, port)
	local fd = io_do.connect(ip)
	if not fd then
		return 599
	end
	header = header or EMPTY
	body = body or ""
	send_request(io_do, fd, method, host, path, header, body)
	local status, header, body, ver = recv_response(io_do, fd)
	io_do.close(fd)
	return status, header, body, ver
end

function client.GET(uri, header)<commit_msg>bugfix: http client use default header<commit_after>local socket = require "socket"
local ssl = require "ssl"
local stream = require "http.stream"
local dns = require "dns"

local client = {}

local function parseurl(url)
	local default = false
	local scheme, host, port, path= string.match(url, "(http[s]-)://([^:/]+):?(%d*)([%w-%.?&=_/]*)")
	if path == "" then
		path = "/"
	end
	if port == "" then
		if scheme == "https" then
			port = "443"
		elseif scheme == "http" then
			port = "80"
		else
			assert(false, "unsupport parse url scheme:" .. scheme)
		end
		default = true
	end
	return scheme, host, port, path, default
end

local function send_request(io_do, fd, method, host, abs, header, body)
	local tmp = ""
	table.insert(header, 1, string.format("%s %s HTTP/1.1", method, abs))
	table.insert(header, string.format("Host: %s", host))
	table.insert(header, string.format("Content-Length: %d", #body))
	table.insert(header, "User-Agent: Silly/0.2")
	table.insert(header, "Connection: keep-alive")
	table.insert(header, "\r\n")
	tmp = table.concat(header, "\r\n")
	tmp = tmp .. body
	io_do.write(fd, tmp)
end

local function recv_response(io_do, fd)
	local readl = function()
		return io_do.readline(fd, "\r\n")
	end
	local readn = function(n)
		return io_do.read(fd, n)
	end
	local status, first, header, body = stream.recv_request(readl, readn)
	if not status then	--disconnected
		return nil
	end
	if status ~= 200 then
		return status
	end
	local ver, status= first:match("HTTP/([%d|.]+)%s+(%d+)")
	return tonumber(status), header, body, ver
end

local function process(uri, method, header, body)
	local ip, io_do
	local scheme, host, port, path, default = parseurl(uri)
	if dns.isdomain(host) then
		ip = dns.query(host)
	else
		ip = host
	end
	if scheme == "https" then
		io_do = ssl
	elseif scheme == "http" then
		io_do = socket
	end
	if not default then
		host = string.format("%s:%s", host, port)
	end
	ip = string.format("%s@%s", ip, port)
	local fd = io_do.connect(ip)
	if not fd then
		return 599
	end
	if not header then
		header = {}
	end
	body = body or ""
	send_request(io_do, fd, method, host, path, header, body)
	local status, header, body, ver = recv_response(io_do, fd)
	io_do.close(fd)
	return status, header, body, ver
end

function client.GET(uri, header)
{'fraction_non_alphanumeric': 0.15367727771679474, 'fraction_numerical': 0.007025246981339188, 'mean_word_length': 4.062222222222222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965285', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1680, 'n_words': 516}