<commit_before>//
//   date  : 2014-06-04
//   author: xjdrew
//

package main

import (
	"bufio"
	"encoding/binary"
	"net"
)

type TunnelPayload struct {
	Linkid uint16
	Data   []byte
}

type Tunnel struct {
	inputCh  chan TunnelPayload
	outputCh chan TunnelPayload
	conn     *net.TCPConn
}

func (self *Tunnel) Put(payload *TunnelPayload) {
	self.inputCh <- *payload
}

func (self *Tunnel) Pop() *TunnelPayload {
	payload, ok := <-self.outputCh
	if !ok {
		return nil
	}
	return &payload
}

// read
func (self *Tunnel) PumpOut() (err error) {
	defer close(self.outputCh)

	var header struct {
		Linkid uint16
		Sz     uint8
	}

	rd := bufio.NewReaderSize(self.conn, 4096)
	for {
		err = binary.Read(rd, binary.LittleEndian, &header)
		if err != nil {
			Error("read tunnel failed:%s", err.Error())
			return
		}

		var data []byte

		// if header.Sz == 0, it's ok too
		data = make([]byte, header.Sz)
		c := 0
		for c < int(header.Sz) {
			var n int
			n, err = rd.Read(data[c:])
			if err != nil {
				Error("read tunnel failed:%s", err.Error())
				return
			}
			c += n
		}

		self.outputCh <- TunnelPayload{header.Linkid, data}
	}
	return
}

// write
func (self *Tunnel) PumpUp() (err error) {
	var header struct {
		Linkid uint16
		Sz     uint8
	}
	for {
		payload := <-self.inputCh

		sz := len(payload.Data)
		if sz > 0xff {
			Panic("receive malformed payload, linkid:%d, sz:%d", payload.Linkid, sz)
			break
		}

		header.Linkid = payload.Linkid
		header.Sz = uint8(sz)
		err = binary.Write(self.conn, binary.LittleEndian, &header)
		if err != nil {
			Error("write tunnel failed:%s", err.Error())
			return
		}

		c := 0
		for c < sz {
			var n int
			n, err = self.conn.Write(payload.Data[c:])
			if err != nil {
				Error("write tunnel failed:%s", err.Error())
				return
			}
			c += n
		}
	}
	return
}

func NewTunnel(conn *net.TCPConn) *Tunnel {
	return &Tunnel{make(chan TunnelPayload, 65535), make(chan TunnelPayload, 65535), conn}
}<commit_msg>use less pointer<commit_after>//
//   date  : 2014-06-04
//   author: xjdrew
//

package main

import (
	"bufio"
	"encoding/binary"
	"net"
)

type TunnelPayload struct {
	Linkid uint16
	Data   []byte
}

type Tunnel struct {
	inputCh  chan *TunnelPayload
	outputCh chan *TunnelPayload
	conn     *net.TCPConn
}

func (self *Tunnel) Put(payload *TunnelPayload) {
	self.inputCh <- payload
}

func (self *Tunnel) Pop() *TunnelPayload {
	payload, ok := <-self.outputCh
	if !ok {
		return nil
	}
	return payload
}

// read
func (self *Tunnel) PumpOut() (err error) {
	defer close(self.outputCh)

	var header struct {
		Linkid uint16
		Sz     uint8
	}

	rd := bufio.NewReaderSize(self.conn, 4096)
	for {
		err = binary.Read(rd, binary.LittleEndian, &header)
		if err != nil {
			Error("read tunnel failed:%s", err.Error())
			return
		}

		var data []byte

		// if header.Sz == 0, it's ok too
		data = make([]byte, header.Sz)
		c := 0
		for c < int(header.Sz) {
			var n int
			n, err = rd.Read(data[c:])
			if err != nil {
				Error("read tunnel failed:%s", err.Error())
				return
			}
			c += n
		}

		self.outputCh <- &TunnelPayload{header.Linkid, data}
	}
	return
}

// write
func (self *Tunnel) PumpUp() (err error) {
	var header struct {
		Linkid uint16
		Sz     uint8
	}
	for {
		payload := <-self.inputCh

		sz := len(payload.Data)
		if sz > 0xff {
			Panic("receive malformed payload, linkid:%d, sz:%d", payload.Linkid, sz)
			break
		}

		header.Linkid = payload.Linkid
		header.Sz = uint8(sz)
		err = binary.Write(self.conn, binary.LittleEndian, &header)
		if err != nil {
			Error("write tunnel failed:%s", err.Error())
			return
		}

		c := 0
		for c < sz {
			var n int
			n, err = self.conn.Write(payload.Data[c:])
			if err != nil {
				Error("write tunnel failed:%s", err.Error())
				return
			}
			c += n
		}
	}
	return
}

func NewTunnel(conn *net.TCPConn) *Tunnel {
	return &Tunnel{make(chan *TunnelPayload, 65535), make(chan *TunnelPayload, 65535), conn}
}
{'fraction_non_alphanumeric': 0.1451448906964921, 'fraction_numerical': 0.017793594306049824, 'mean_word_length': 2.8999008919722495, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4243392', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1488, 'n_words': 400}