<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package compress

const (
	filterScale = 4096
	filterShift = 5
	cdfScale    = 1 << (16 - 3)
	cdfRate     = 5
	CDFScale    = cdfScale
)

type CDF struct {
	CDF    []uint16
	Mixin  [][]uint16
	Verify bool
}

func NewCDF(size int) *CDF {
	if size != 256 {
		panic("size is not 256")
	}
	cdf, mixin := make([]uint16, size+1), make([][]uint16, size)

	sum := 0
	for i := range cdf {
		cdf[i] = uint16(sum)
		sum += 32
	}

	for i := range mixin {
		sum, m := 0, make([]uint16, size+1)
		for j := range m {
			m[j] = uint16(sum)
			sum++
			if j == i {
				sum += cdfScale - size
			}
		}
		mixin[i] = m
	}

	return &CDF{
		CDF:    cdf,
		Mixin:  mixin,
		Verify: true,
	}
}

func (c *CDF) Len() int {
	return len(c.CDF)
}

func (c *CDF) Less(i, j int) bool {
	cdf := c.CDF
	return cdf[i] < cdf[j]
}

func (c *CDF) Swap(i, j int) {
	cdf := c.CDF
	cdf[i], cdf[j] = cdf[j], cdf[i]
}

func (c *CDF) Update(s int) {
	cdf, mixin := c.CDF, c.Mixin[s]
	size := len(cdf) - 1

	if c.Verify {
		for i := 1; i < size; i++ {
			a, b := int(cdf[i]), int(mixin[i])
			if a < 0 {
				panic("a is less than zero")
			}
			if b < 0 {
				panic("b is less than zero")
			}
			c := (b - a)
			if c >= 0 {
				c >>= cdfRate
				c = a + c
			} else {
				c = -c
				c >>= cdfRate
				c = a - c
			}
			if c < 0 {
				panic("c is less than zero")
			}
			cdf[i] = uint16(c)
		}
		if cdf[size] != cdfScale {
			panic("cdf scale is incorrect")
		}
		for i := 1; i < len(cdf); i++ {
			if cdf[i] <= cdf[i-1] {
				panic("invalid cdf")
			}
		}
		return
	}

	for i := 1; i < size; i++ {
		a, b := int(cdf[i]), int(mixin[i])
		c := (b - a)
		if c >= 0 {
			c >>= cdfRate
			c = a + c
		} else {
			c = -c
			c >>= cdfRate
			c = a - c
		}
		cdf[i] = uint16(c)<commit_msg>Improved performance<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package compress

const (
	filterScale = 4096
	filterShift = 5
	cdfScale    = 1 << (16 - 3)
	cdfRate     = 5
	CDFScale    = cdfScale
)

type CDF struct {
	CDF    []uint16
	Mixin  [][]uint16
	Verify bool
}

func NewCDF(size int) *CDF {
	if size != 256 {
		panic("size is not 256")
	}
	cdf, mixin := make([]uint16, size+1), make([][]uint16, size)

	sum := 0
	for i := range cdf {
		cdf[i] = uint16(sum)
		sum += 32
	}

	for i := range mixin {
		sum, m := 0, make([]uint16, size+1)
		for j := range m {
			m[j] = uint16(sum)
			sum++
			if j == i {
				sum += cdfScale - size
			}
		}
		mixin[i] = m
	}

	return &CDF{
		CDF:    cdf,
		Mixin:  mixin,
		Verify: true,
	}
}

func (c *CDF) Len() int {
	return len(c.CDF)
}

func (c *CDF) Less(i, j int) bool {
	cdf := c.CDF
	return cdf[i] < cdf[j]
}

func (c *CDF) Swap(i, j int) {
	cdf := c.CDF
	cdf[i], cdf[j] = cdf[j], cdf[i]
}

func (c *CDF) Update(s int) {
	cdf, mixin := c.CDF, c.Mixin[s]
	size := len(cdf) - 1

	if c.Verify {
		for i := 1; i < size; i++ {
			a, b := int(cdf[i]), int(mixin[i])
			if a < 0 {
				panic("a is less than zero")
			}
			if b < 0 {
				panic("b is less than zero")
			}
			/*c := (b - a)
			if c >= 0 {
				c >>= cdfRate
				c = a + c
			} else {
				c = -c
				c >>= cdfRate
				c = a - c
			}
			if c < 0 {
				panic("c is less than zero")
			}*/
			cdf[i] = uint16(a + ((b - a) >> cdfRate))
		}
		if cdf[size] != cdfScale {
			panic("cdf scale is incorrect")
		}
		for i := 1; i < len(cdf); i++ {
			if cdf[i] <= cdf[i-1] {
				panic("invalid cdf")
			}
		}
		return
	}

	for i := 1; i < size; i++ {
		a, b := int(cdf[i]), int(mixin[i])
		/*c := (b - a)
		if c >= 0 {
			c >>= cdfRate
			c = a + c
		} else {
			c = -c
			c >>= cdfRate
			c = a - c
		}*/
		cdf[i] = uint16(a + ((b - a) >> cdfRate))
{'fraction_non_alphanumeric': 0.18223938223938224, 'fraction_numerical': 0.027799227799227798, 'mean_word_length': 2.0988835725677832, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4677777', 'n_tokens_mistral': 2066, 'n_tokens_neox': 1777, 'n_words': 528}