<commit_before>package bzip2

import (
	"bytes"
	"errors"
	"hash/crc32"
)

const (
	blockMagic = 0x314159265359 // pi
)

var (
	errBlockSizeReached = errors.New("bzip2: Block size reached")
)

// block handles the compression of data for a single block of a given size.
type block struct {
	buf  *bytes.Buffer
	size int
	crc  uint32
}

// newBlock creates a compression block for data up to the given size.
func newBlock(size int) *block {
	return &block{buf: bytes.NewBuffer(make([]byte, 0, size)), size: size}
}

// Len returns the number of bytes written to the block.
func (b *block) Len() int {
	return b.buf.Len()
}

// Write writes p to the blocks buffer. If writing p exceeds the blocks size
// only the bytes that can fit will be written and errBlockSizeReached is
// returned.
func (b *block) Write(p []byte) (int, error) {
	exceedsSize := false

	// Do the initial RLE step on demand since RLE can actually make p grow.
	// This ensures that the block size doesn't end up more than b.size
	// after RLE.
	data := rlEncode(p)
	if b.buf.Len()+len(data) > b.size {
		exceedsSize = true
		data = data[:b.size-b.buf.Len()]
	}

	n, err := b.buf.Write(data)
	if err == nil {
		b.crc = crc32.Update(b.crc, crc32.IEEETable, p)

		if exceedsSize || b.buf.Len() == b.size {
			err = errBlockSizeReached
		}
	}

	p = p[:rlIndexOf(data, n-1)+1]
	return len(p), err
}<commit_msg>Fix block crc for overwrites<commit_after>package bzip2

import (
	"bytes"
	"errors"
	"hash/crc32"
)

const (
	blockMagic = 0x314159265359 // pi
)

var (
	errBlockSizeReached = errors.New("bzip2: Block size reached")
)

// block handles the compression of data for a single block of a given size.
type block struct {
	buf  *bytes.Buffer
	size int
	crc  uint32
}

// newBlock creates a compression block for data up to the given size.
func newBlock(size int) *block {
	return &block{buf: bytes.NewBuffer(make([]byte, 0, size)), size: size}
}

// Len returns the number of bytes written to the block.
func (b *block) Len() int {
	return b.buf.Len()
}

// Write writes p to the blocks buffer. If writing p exceeds the blocks size
// only the bytes that can fit will be written and errBlockSizeReached is
// returned.
func (b *block) Write(p []byte) (int, error) {
	exceedsSize := false

	// Do the initial RLE step on demand since RLE can actually make p grow.
	// This ensures that the block size doesn't end up more than b.size
	// after RLE.
	data := rlEncode(p)
	if b.buf.Len()+len(data) > b.size {
		exceedsSize = true
		data = data[:b.size-b.buf.Len()]
	}

	n, err := b.buf.Write(data)
	p = p[:rlIndexOf(data, n-1)+1]

	if err == nil {
		b.crc = crc32.Update(b.crc, crc32.IEEETable, p)

		if exceedsSize || b.buf.Len() == b.size {
			err = errBlockSizeReached
		}
	}

	return len(p), err
}
{'fraction_non_alphanumeric': 0.12427745664739884, 'fraction_numerical': 0.01878612716763006, 'mean_word_length': 3.3401253918495297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150113', 'n_tokens_mistral': 1057, 'n_tokens_neox': 998, 'n_words': 352}