<commit_before>package wstest

import (
	"net"
	"time"
)

// conn is a connection for testing, implementing the net.Conn interface
type conn struct {
	in     <-chan []byte
	out    chan<- []byte
	local  net.Addr
	remote net.Addr
}

// newConnPair returns two connections, paired by channels.
// any message written into the first will be read in the second
// and vice-versa.
func newConnPair(connCapacity int) (server, client net.Conn) {
	var (
		s2c   = make(chan []byte, connCapacity)
		c2s   = make(chan []byte, connCapacity)
		cAddr = &address{"tcp", "127.0.0.1:12345"}
		sAddr = &address{"tcp", "8.8.8.8:12346"}
	)

	server = &conn{in: c2s, out: s2c, local: sAddr, remote: cAddr}
	client = &conn{in: s2c, out: c2s, local: cAddr, remote: sAddr}
	return
}

func (c *conn) Read(b []byte) (n int, err error) {
	read := <-c.in
	n = copy(b, read)
	return
}

func (c *conn) Write(b []byte) (n int, err error) {
	c.out <- b
	return len(b), nil
}

func (c *conn) Close() error {
	close(c.out)
	return nil
}

func (c *conn) LocalAddr() net.Addr { return c.local }

func (c *conn) RemoteAddr() net.Addr { return c.remote }

func (c *conn) SetDeadline(t time.Time) error { return nil }

func (c *conn) SetReadDeadline(t time.Time) error { return nil }

func (c *conn) SetWriteDeadline(t time.Time) error { return nil }

type address struct {
	network string
	address string
}

func (a *address) Network() string { return a.network }

func (a *address) String() string { return a.address }
<commit_msg>Add buffer to connection read<commit_after>package wstest

import (
	"net"
	"time"
)

// conn is a connection for testing, implementing the net.Conn interface
type conn struct {
	in      <-chan []byte
	out     chan<- []byte
	local   net.Addr
	remote  net.Addr
	readBuf []byte
}

// newConnPair returns two connections, paired by channels.
// any message written into the first will be read in the second
// and vice-versa.
func newConnPair(connCapacity int) (server, client net.Conn) {
	var (
		s2c   = make(chan []byte, connCapacity)
		c2s   = make(chan []byte, connCapacity)
		cAddr = &address{"tcp", "127.0.0.1:12345"}
		sAddr = &address{"tcp", "8.8.8.8:12346"}
	)

	server = &conn{in: c2s, out: s2c, local: sAddr, remote: cAddr}
	client = &conn{in: s2c, out: c2s, local: cAddr, remote: sAddr}
	return
}

func (c *conn) Read(b []byte) (n int, err error) {
	if len(c.readBuf) == 0 {
		c.readBuf = <-c.in
	}
	n = copy(b, c.readBuf)
	c.readBuf = c.readBuf[n:]
	return
}

func (c *conn) Write(b []byte) (n int, err error) {
	c.out <- b
	return len(b), nil
}

func (c *conn) Close() error {
	close(c.out)
	return nil
}

func (c *conn) LocalAddr() net.Addr { return c.local }

func (c *conn) RemoteAddr() net.Addr { return c.remote }

func (c *conn) SetDeadline(t time.Time) error { return nil }

func (c *conn) SetReadDeadline(t time.Time) error { return nil }

func (c *conn) SetWriteDeadline(t time.Time) error { return nil }

type address struct {
	network string
	address string
}

func (a *address) Network() string { return a.network }

func (a *address) String() string { return a.address }

{'fraction_non_alphanumeric': 0.1436893203883495, 'fraction_numerical': 0.01715210355987055, 'mean_word_length': 3.6134328358208956, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3163155', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1145, 'n_words': 381}