<commit_before>package restruct

import (
	"encoding/binary"
	"fmt"
	"math"
	"reflect"

)

// Unpacker is a type capable of unpacking a binary representation of itself
// into a native representation. The Unpack function is expected to consume
// a number of bytes from the buffer, then return a slice of the remaining
// bytes in the buffer. You may use a pointer receiver even if the type is
// used by value.
type Unpacker interface {
	Unpack(buf []byte, order binary.ByteOrder) ([]byte, error)
}

type decoder struct {
	order      binary.ByteOrder
	buf        []byte
	struc      reflect.Value
	sfields    []field
	bitCounter uint8
}

func (d *decoder) readBits(f field, outputLength uint8) (output []byte) {

	output = make([]byte, outputLength)

	if f.BitSize == 0 {
		// Having problems with complex64 type ... so we asume we want to read all
		// f.BitSize = uint8(f.Type.Bits())
		f.BitSize = 8 * outputLength
	}

	// originPos: Original position of the first bit in the first byte
	var originPos uint8 = 8 - d.bitCounter

	// destPos: Destination position ( in the result ) of the first bit in the first byte
	var destPos uint8 = f.BitSize % 8
	if destPos == 0 {
		destPos = 8
	}

	// numBytes: number of complete bytes to hold the result
	var numBytes uint8 = f.BitSize / 8

	// numBits: number of remaining bits in the first non-complete byte of the result
	var numBits uint8 = f.BitSize % 8

	// number of positions we have to shift the bytes to get the result
	var shift uint8 = (uint8(math.Abs(float64(originPos - destPos)))) % 8

	var outputInitialIdx uint8 = outputLength - numBytes
	if numBits > 0 {
		outputInitialIdx = outputInitialIdx - 1
	}

	if originPos < destPos { // shift left
		var idx uint8 = 0
		for outIdx := outputInitialIdx; outIdx < outputLength; outIdx++ {
			// TODO: Control the number of bytes of d.buf ... we need to read ahead
			var carry uint8 = d.buf[idx+1] >> (8 - shift)
			output[outIdx] = (d.buf[idx] << shift) | carry
			idx++
		}
	} else { // originPos >= destPos => shift right
		var idx uint8 = 0

		// carry : is a little bit tricky in this case because of the first case
		// when idx == 0 and there is no carry at all
		carry := func(idx uint8) uint8 {
			if idx == 0 {
				return 0x00
			}
			return (d.buf[idx-1] << (8 - shift))
		}

		for outIdx := outputInitialIdx; outIdx < outputLength; outIdx++ {
			output[outIdx] = (d.buf[idx] >> shift) | carry(idx)
			idx++
		}
	}

	// here the output is calculated ... but the first byte may have some extra bits
	// therefore we apply a mask to erase those unaddressable bits
	output[outputInitialIdx] &= ((0x01 << destPos) - 1)

	// now we need to update the head of the incoming buffer and the bitCounter
	d.bitCounter = (d.bitCounter + f.BitSize) % 8

	// move the head to the next non-complete byte used
	headerUpdate := func() uint8 {
		if (d.bitCounter == 0) && ((f.BitSize % 8) != 0) {
			return (numBytes + 1)
		}
		return numBytes
	}

	d.buf = d.buf[headerUpdate():]

	return
}

func (d *decoder) read8(f field) uint8 {
	rawdata := d.readBits(f, 1)
	return uint8(rawdata[0])
}

func (d *decoder) read16(f field) uint16 {
	rawdata := d.readBits(f, 2)
	return d.order.Uint16(rawdata)
}

func (d *decoder) read32(f field) uint32 {
	rawdata := d.readBits(f, 4)
	return d.order.Uint32(rawdata)
}

func (d *decoder) read64(f field) uint64 {
	rawdata := d.readBits(f, 8)
	return d.order.Uint64(rawdata)
}

func (d *decoder) readS8(f field) int8 { return int8(d.read8(f)) }
<commit_msg>Address Readability issues in readBits<commit_after>package restruct

import (
	"encoding/binary"
	"fmt"
	"math"
	"reflect"
)

// Unpacker is a type capable of unpacking a binary representation of itself
// into a native representation. The Unpack function is expected to consume
// a number of bytes from the buffer, then return a slice of the remaining
// bytes in the buffer. You may use a pointer receiver even if the type is
// used by value.
type Unpacker interface {
	Unpack(buf []byte, order binary.ByteOrder) ([]byte, error)
}

type decoder struct {
	order      binary.ByteOrder
	buf        []byte
	struc      reflect.Value
	sfields    []field
	bitCounter uint8
}

func (d *decoder) readBits(f field, outputLength uint8) []byte {
	output := make([]byte, outputLength)

	if f.BitSize == 0 {
		// Having problems with complex64 type ... so we asume we want to read all
		// f.BitSize = uint8(f.Type.Bits())
		f.BitSize = 8 * outputLength
	}

	// originPos: Original position of the first bit in the first byte
	originPos := 8 - d.bitCounter

	// destPos: Destination position ( in the result ) of the first bit in the first byte
	destPos := f.BitSize % 8
	if destPos == 0 {
		destPos = 8
	}

	// numBytes: number of complete bytes to hold the result
	numBytes := f.BitSize / 8

	// numBits: number of remaining bits in the first non-complete byte of the result
	numBits := f.BitSize % 8

	// number of positions we have to shift the bytes to get the result
	shift := (originPos - destPos) % 8

	outputInitialIdx := outputLength - numBytes
	if numBits > 0 {
		outputInitialIdx = outputInitialIdx - 1
	}
	o := output[outputInitialIdx:]
	if originPos < destPos { // shift left
		for idx := range o {
			// TODO: Control the number of bytes of d.buf ... we need to read ahead
			carry := d.buf[idx+1] >> (8 - shift)
			o[idx] = (d.buf[idx] << shift) | carry
		}
	} else { // originPos >= destPos => shift right
		// carry : is a little bit tricky in this case because of the first case
		// when idx == 0 and there is no carry at all
		carry := func(idx int) uint8 {
			if idx == 0 {
				return 0x00
			}
			return (d.buf[idx-1] << (8 - shift))
		}

		for idx := range o {
			o[idx] = (d.buf[idx] >> shift) | carry(idx)
		}
	}

	// here the output is calculated ... but the first byte may have some extra bits
	// therefore we apply a mask to erase those unaddressable bits
	output[outputInitialIdx] &= ((0x01 << destPos) - 1)

	// now we need to update the head of the incoming buffer and the bitCounter
	d.bitCounter = (d.bitCounter + f.BitSize) % 8

	// move the head to the next non-complete byte used
	headerUpdate := func() uint8 {
		if (d.bitCounter == 0) && ((f.BitSize % 8) != 0) {
			return (numBytes + 1)
		}
		return numBytes
	}

	d.buf = d.buf[headerUpdate():]
	return output
}

func (d *decoder) read8(f field) uint8 {
	rawdata := d.readBits(f, 1)
	return uint8(rawdata[0])
}

func (d *decoder) read16(f field) uint16 {
	rawdata := d.readBits(f, 2)
	return d.order.Uint16(rawdata)
}

func (d *decoder) read32(f field) uint32 {
	rawdata := d.readBits(f, 4)
	return d.order.Uint32(rawdata)
}

func (d *decoder) read64(f field) uint64 {
	rawdata := d.readBits(f, 8)
	return d.order.Uint64(rawdata)
}

func (d *decoder) readS8(f field) int8 { return int8(d.read8(f)) }

{'fraction_non_alphanumeric': 0.11465721040189125, 'fraction_numerical': 0.02171985815602837, 'mean_word_length': 3.1451316595223515, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4836512', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2256, 'n_words': 856}