<commit_before>package lzma

import (
	"io"
)

// Maximum and minimum values for individual parameters.
const (
	MinLC       = 0
	MaxLC       = 8
	MinLP       = 0
	MaxLP       = 4
	MinPB       = 0
	MaxPB       = 4
	MinDictSize = 1 << 12
	MaxDictSize = 1<<32 - 1
)

// Properties contains the parametes lc, lp and pb.
type Properties struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
}

// verifyProperties checks the argument for any errors.
func verifyProperties(p *Properties) error {
	if !(MinLC <= p.LC && p.LC <= MaxLC) {
		return newError("lc out of range")
	}
	if !(MinLP <= p.LP && p.LP <= MaxLC) {
		return newError("lp out of range")
	}
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return newError("pb out of range")
	}
	return nil
}

// props reads a single properties byte.
func (p *Properties) Decode(b byte) error {
	x := int(b)
	p.LC = x % 9
	x /= 9
	p.LP = x % 5
	p.PB = x / 5
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return newError("PB out of range")
	}
	return nil
}

// Encodes the properties in a single byte.
func (p *Properties) Byte() byte {
	return byte((p.PB*5+p.LP)*9 + p.LC)
}

// Parameters contain all information required to decode or encode an LZMA
// stream.
//
// The DictSize will be limited by MaxInt32 on 32-bit platforms.
type Parameters struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
	// size of the dictionary in bytes
	DictSize uint32
	// size of uncompressed data in bytes
	Size int64
	// header includes unpacked size
	SizeInHeader bool
	// end-of-stream marker requested
	EOS bool
}

// Properties returns lc, lp and pb as Properties value.
func (p *Parameters) Properties() *Properties {
	return &Properties{LC: p.LC, LP: p.LP, PB: p.PB}
}

// verifyParameters checks parameters for errors.
func verifyParameters(p *Parameters) error {
	if p == nil {
		return newError("parameters must be non-nil")
	}
	if err := verifyProperties(p.Properties()); err != nil {
		return err
	}
	if !(MinDictSize <= p.DictSize && p.DictSize <= MaxDictSize) {
		return newError("DictSize out of range")
	}
	hlen := int(p.DictSize)
	if hlen < 0 {
		return newError("DictSize cannot be converted into int")
	}
	if p.Size < 0 {
		return newError("length must not be negative")
	}
	return nil
}

// getUint32LE reads an uint32 integer from a byte slize
func getUint32LE(b []byte) uint32 {
	x := uint32(b[3]) << 24
	x |= uint32(b[2]) << 16
	x |= uint32(b[1]) << 8
	x |= uint32(b[0])
	return x
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// putUint32LE puts an uint32 integer into a byte slice that must have at least
// a lenght of 4 bytes.
func putUint32LE(b []byte, x uint32) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
}

// putUint64LE puts the uint64 value into the byte slice as little endian
// value. The byte slice b must have at least place for 8 bytes.
func putUint64LE(b []byte, x uint64) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
	b[4] = byte(x >> 32)
	b[5] = byte(x >> 40)
	b[6] = byte(x >> 48)
	b[7] = byte(x >> 56)
}

// readHeader reads the classic LZMA header.
func readHeader(r io.Reader) (p *Parameters, err error) {
	b := make([]byte, 13)
	_, err = io.ReadFull(r, b)
	if err != nil {
		return nil, err
	}
	p = new(Parameters)
	var props Properties
	if err = props.Decode(b[0]); err != nil {
		return nil, err
	}
	p.LC, p.LP, p.PB = props.LC, props.LP, props.PB
	p.DictSize = getUint32LE(b[1:])
	if p.DictSize < MinDictSize {
		// The LZMA specification makes the following recommendation.
		p.DictSize = MinDictSize
	}
	u := getUint64LE(b[5:])
	if u == noHeaderLen {
		p.Size = 0
		p.EOS = true
		p.SizeInHeader = false
		return p, nil
	}
	p.Size = int64(u)
	if p.Size < 0 {
		return nil, newError(
			"unpack length in header not supported by int64")
	}
	p.EOS = false
	p.SizeInHeader = true
	return p, nil
}

// writeHeader writes the header for classic LZMA files.
func writeHeader(w io.Writer, p *Parameters) error {
	var err error
	if err = verifyParameters(p); err != nil {
		return err
	}
	b := make([]byte, 13)
	b[0] = p.Properties().Byte()
	putUint32LE(b[1:5], p.DictSize)
	var l uint64
	if p.SizeInHeader {
		l = uint64(p.Size)
	} else {
		l = noHeaderLen
	}
	putUint64LE(b[5:], l)
	_, err = w.Write(b)
	return err
}<commit_msg>lzma: Properties value for LC, LP and PB implemented<commit_after>package lzma

import (
	"io"
)

// Maximum and minimum values for individual parameters.
const (
	MinLC       = 0
	MaxLC       = 8
	MinLP       = 0
	MaxLP       = 4
	MinPB       = 0
	MaxPB       = 4
	MinDictSize = 1 << 12
	MaxDictSize = 1<<32 - 1
)

// Properties contains the parametes lc, lp and pb.
type Properties byte

func NewProperties(lc, lp, pb int) (p Properties, err error) {
	if err = verifyProperties(lc, lp, pb); err != nil {
		return
	}
	return Properties((pb*5+lp)*9 + lc), nil
}

func (p Properties) LC() int {
	return int(p) % 9
}

func (p Properties) LP() int {
	return (int(p) / 9) % 5
}

func (p Properties) PB() int {
	return int(p) / 45
}

// verifyProperties checks the argument for any errors.
func verifyProperties(lc, lp, pb int) error {
	if !(MinLC <= lc && lc <= MaxLC) {
		return newError("lc out of range")
	}
	if !(MinLP <= lp && lp <= MaxLC) {
		return newError("lp out of range")
	}
	if !(MinPB <= pb && pb <= MaxPB) {
		return newError("pb out of range")
	}
	return nil
}

// Parameters contain all information required to decode or encode an LZMA
// stream.
//
// The DictSize will be limited by MaxInt32 on 32-bit platforms.
type Parameters struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
	// size of the dictionary in bytes
	DictSize uint32
	// size of uncompressed data in bytes
	Size int64
	// header includes unpacked size
	SizeInHeader bool
	// end-of-stream marker requested
	EOS bool
}

// Properties returns lc, lp and pb as Properties value.
func (p *Parameters) Properties() Properties {
	props, err := NewProperties(p.LC, p.LP, p.PB)
	if err != nil {
		panic(err)
	}
	return props
}

// verifyParameters checks parameters for errors.
func verifyParameters(p *Parameters) error {
	if p == nil {
		return newError("parameters must be non-nil")
	}
	if err := verifyProperties(p.LC, p.LP, p.PB); err != nil {
		return err
	}
	if !(MinDictSize <= p.DictSize && p.DictSize <= MaxDictSize) {
		return newError("DictSize out of range")
	}
	hlen := int(p.DictSize)
	if hlen < 0 {
		return newError("DictSize cannot be converted into int")
	}
	if p.Size < 0 {
		return newError("length must not be negative")
	}
	return nil
}

// getUint32LE reads an uint32 integer from a byte slize
func getUint32LE(b []byte) uint32 {
	x := uint32(b[3]) << 24
	x |= uint32(b[2]) << 16
	x |= uint32(b[1]) << 8
	x |= uint32(b[0])
	return x
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// putUint32LE puts an uint32 integer into a byte slice that must have at least
// a lenght of 4 bytes.
func putUint32LE(b []byte, x uint32) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
}

// putUint64LE puts the uint64 value into the byte slice as little endian
// value. The byte slice b must have at least place for 8 bytes.
func putUint64LE(b []byte, x uint64) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
	b[4] = byte(x >> 32)
	b[5] = byte(x >> 40)
	b[6] = byte(x >> 48)
	b[7] = byte(x >> 56)
}

// readHeader reads the classic LZMA header.
func readHeader(r io.Reader) (p *Parameters, err error) {
	b := make([]byte, 13)
	_, err = io.ReadFull(r, b)
	if err != nil {
		return nil, err
	}
	p = new(Parameters)
	props := Properties(b[0])
	p.LC, p.LP, p.PB = props.LC(), props.LP(), props.PB()
	if err = verifyProperties(p.LC, p.LP, p.PB); err != nil {
		return nil, err
	}
	p.DictSize = getUint32LE(b[1:])
	if p.DictSize < MinDictSize {
		// The LZMA specification makes the following recommendation.
		p.DictSize = MinDictSize
	}
	u := getUint64LE(b[5:])
	if u == noHeaderLen {
		p.Size = 0
		p.EOS = true
		p.SizeInHeader = false
		return p, nil
	}
	p.Size = int64(u)
	if p.Size < 0 {
		return nil, newError(
			"unpack length in header not supported by int64")
	}
	p.EOS = false
	p.SizeInHeader = true
	return p, nil
}

// writeHeader writes the header for classic LZMA files.
func writeHeader(w io.Writer, p *Parameters) error {
	var err error
	if err = verifyParameters(p); err != nil {
		return err
	}
	b := make([]byte, 13)
	b[0] = byte(p.Properties())
	putUint32LE(b[1:5], p.DictSize)
	var l uint64
	if p.SizeInHeader {
		l = uint64(p.Size)
	} else {
		l = noHeaderLen
	}
	putUint64LE(b[5:], l)
	_, err = w.Write(b)
	return err
}
{'fraction_non_alphanumeric': 0.1417902485863651, 'fraction_numerical': 0.038088125466766244, 'mean_word_length': 3.1150131694468834, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2926165', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3564, 'n_words': 1201}