<commit_before>// Package http2client is a HTTP/2 client library.
package http2client

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
	"net"
)

type Http2Client struct {
	host   string
	port   int
	conn   net.Conn
	framer *http2.Framer
}

func New() *Http2Client {
	return &Http2Client{}
}

func (h2c *Http2Client) Connect(host string, port int) (string, error) {
	if h2c.isConnected() {
		return "", fmt.Errorf("Already connected to %v:%v.", h2c.host, h2c.port)
	}
	hostAndPort := fmt.Sprintf("%v:%v", host, port)
	conn, err := tls.Dial("tcp", hostAndPort, &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{http2.NextProtoTLS},
	})
	if err != nil {
		return "", fmt.Errorf("Failed to connect to %v: %v", hostAndPort, err.Error())
	}
	_, err = conn.Write([]byte(http2.ClientPreface))
	if err != nil {
		return "", fmt.Errorf("Failed to write client preface to %v: %v", hostAndPort, err.Error())
	}
	framer := http2.NewFramer(conn, conn)
	err = framer.WriteSettings()
	if err != nil {
		return "", fmt.Errorf("Failed to write initial settings frame to %v: %v", hostAndPort, err.Error())
	}
	frame, err := framer.ReadFrame()
	if err != nil || frame.Header().Type != http2.FrameSettings {
		return "", fmt.Errorf("Failed to read initial settings frame from %v: %v", hostAndPort, err.Error())
	}
	h2c.conn = conn
	h2c.framer = framer
	h2c.host = host
	h2c.port = port
	return "", nil
}

func (h2c *Http2Client) Get(path string) (string, error) {
	if !h2c.isConnected() {
		return "", fmt.Errorf("Not connected.")
	}
	err := h2c.framer.WriteHeaders(http2.HeadersFrameParam{
		StreamID:      1,
		BlockFragment: makeGet(h2c.host),
		EndStream:     true,
		EndHeaders:    true,
	})
	if err != nil {
		return "", fmt.Errorf("Failed to write HEADERS frame to %v: %v", h2c.host, err.Error())
	}
	for {
		frame, err := h2c.framer.ReadFrame()
		if err != nil {
			return "", fmt.Errorf("Failed to read next frame: %v", err.Error())
		}
		if frame.Header().Type == http2.FrameHeaders {
			// TODO: Handle 404, etc.
		}
		if frame.Header().Type == http2.FrameData {
			dataFrame, ok := frame.(*http2.DataFrame)
			if !ok {
				return "", fmt.Errorf("ERRO: Incompatable version of github.com/bradfitz/http2")
			}
			return string(dataFrame.Data()), nil
		}
	}
}

func (h2c *Http2Client) isConnected() bool {
	return h2c.conn != nil || h2c.framer != nil || h2c.host != "" || h2c.port != 0
}

func makeGet(host string) []byte {

	var buf bytes.Buffer
	encoder := hpack.NewEncoder(&buf)

	encoder.WriteField(hpack.HeaderField{Name: ":authority", Value: host})
	encoder.WriteField(hpack.HeaderField{Name: ":method", Value: "GET"})
	encoder.WriteField(hpack.HeaderField{Name: ":path", Value: "/index.html"})
	encoder.WriteField(hpack.HeaderField{Name: ":scheme", Value: "https"})

	return buf.Bytes()
}<commit_msg>Remove hard-coded /index.html from h2c get<commit_after>// Package http2client is a HTTP/2 client library.
package http2client

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
	"net"
)

type Http2Client struct {
	host   string
	port   int
	conn   net.Conn
	framer *http2.Framer
}

func New() *Http2Client {
	return &Http2Client{}
}

func (h2c *Http2Client) Connect(host string, port int) (string, error) {
	if h2c.isConnected() {
		return "", fmt.Errorf("Already connected to %v:%v.", h2c.host, h2c.port)
	}
	hostAndPort := fmt.Sprintf("%v:%v", host, port)
	conn, err := tls.Dial("tcp", hostAndPort, &tls.Config{
		InsecureSkipVerify: true,
		NextProtos:         []string{http2.NextProtoTLS},
	})
	if err != nil {
		return "", fmt.Errorf("Failed to connect to %v: %v", hostAndPort, err.Error())
	}
	_, err = conn.Write([]byte(http2.ClientPreface))
	if err != nil {
		return "", fmt.Errorf("Failed to write client preface to %v: %v", hostAndPort, err.Error())
	}
	framer := http2.NewFramer(conn, conn)
	err = framer.WriteSettings()
	if err != nil {
		return "", fmt.Errorf("Failed to write initial settings frame to %v: %v", hostAndPort, err.Error())
	}
	frame, err := framer.ReadFrame()
	if err != nil || frame.Header().Type != http2.FrameSettings {
		return "", fmt.Errorf("Failed to read initial settings frame from %v: %v", hostAndPort, err.Error())
	}
	h2c.conn = conn
	h2c.framer = framer
	h2c.host = host
	h2c.port = port
	return "", nil
}

func (h2c *Http2Client) Get(path string) (string, error) {
	if !h2c.isConnected() {
		return "", fmt.Errorf("Not connected.")
	}
	err := h2c.framer.WriteHeaders(http2.HeadersFrameParam{
		StreamID:      1,
		BlockFragment: makeGet(h2c.host, path),
		EndStream:     true,
		EndHeaders:    true,
	})
	if err != nil {
		return "", fmt.Errorf("Failed to write HEADERS frame to %v: %v", h2c.host, err.Error())
	}
	for {
		frame, err := h2c.framer.ReadFrame()
		if err != nil {
			return "", fmt.Errorf("Failed to read next frame: %v", err.Error())
		}
		if frame.Header().Type == http2.FrameHeaders {
			// TODO: Handle 404, etc.
		}
		if frame.Header().Type == http2.FrameData {
			dataFrame, ok := frame.(*http2.DataFrame)
			if !ok {
				return "", fmt.Errorf("ERRO: Incompatable version of github.com/bradfitz/http2")
			}
			return string(dataFrame.Data()), nil
		}
	}
}

func (h2c *Http2Client) isConnected() bool {
	return h2c.conn != nil || h2c.framer != nil || h2c.host != "" || h2c.port != 0
}

func makeGet(host, path string) []byte {

	var buf bytes.Buffer
	encoder := hpack.NewEncoder(&buf)

	encoder.WriteField(hpack.HeaderField{Name: ":authority", Value: host})
	encoder.WriteField(hpack.HeaderField{Name: ":method", Value: "GET"})
	encoder.WriteField(hpack.HeaderField{Name: ":path", Value: path})
	encoder.WriteField(hpack.HeaderField{Name: ":scheme", Value: "https"})

	return buf.Bytes()
}
{'fraction_non_alphanumeric': 0.16597438560055383, 'fraction_numerical': 0.015749394254067153, 'mean_word_length': 3.9690455717970767, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599275', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2128, 'n_words': 521}