<commit_before>package reedsolomon

import "errors"

type matrix []byte

func newMatrix(rows, cols int) matrix {
	m := make([]byte, rows*cols)
	return m
}

func genEncMatrixCauchy(data, parity int) matrix {
	rows := data + parity
	cols := data
	m := newMatrix(rows, cols)
	for i := 0; i < cols; i++ {
		m[i*data+i] = byte(1)
	}

	p := data * data
	for i := cols; i < rows; i++ {
		for j := 0; j < cols; j++ {
			d := i ^ j
			a := inverseTbl[d]
			m[p] = byte(a)
			p++
		}
	}
	return m
}

func genVandMatrix(rows, cols int) matrix {
	m := newMatrix(rows, cols)
	for i := 0; i < rows; i++ {
		for j := 0; j < cols; j++ {
			m[i*cols+j] = gfExp(byte(i), j)
		}
	}
	return m
}

func gfExp(b byte, n int) byte {
	if n == 0 {
		return 1
	}
	if b == 0 {
		return 0
	}
	a := logTbl[b]
	ret := int(a) * n
	for ret >= 255 {
		ret -= 255
	}
	return byte(expTbl[ret])
}

func genEncMatrixVand(data, parity int) (matrix, error) {
	rows := data + parity
	vm := genVandMatrix(rows, data)
	top := newMatrix(data, data)
	copy(top, vm[:data*data])
	topI, err := top.invert(data)
	if err != nil {
		return nil, err
	}
	return vm.mul(topI, data+parity, data), nil
}

func (m matrix) invert(n int) (matrix, error) {
	raw := newMatrix(n, 2*n)
	// [m] -> [m|I]
	for i := 0; i < n; i++ {
		t := i * n
		copy(raw[2*t:2*t+n], m[t:t+n])
		raw[2*t+i+n] = byte(1)
	}
	// [m|I] -> [I|m']
	err := raw.gaussJordan(n, 2*n)
	if err != nil {
		return nil, err
	}
	// [I|m'] -> [m']
	return raw.subMatrix(n), nil
}

func (m matrix) swap(i, j, n int) {
	for k := 0; k < n; k++ {
		m[i*n+k], m[j*n+k] = m[j*n+k], m[i*n+k]
	}
}

func gfMul(a, b byte) byte {
	return mulTbl[a][b]
}

var errSingular = errors.New("rs.invert: matrix is singular")

func (m matrix) gaussJordan(rows, columns int) error {
	for r := 0; r < rows; r++ {
		if m[2*r*rows+r] == 0 {
			for rowBelow := r + 1; rowBelow < rows; rowBelow++ {
				if m[2*rowBelow*rows+r] != 0 {
					m.swap(r, rowBelow, 2*rows)
					break
				}
			}
		}
		if m[2*r*rows+r] == 0 {
			return errSingular
		}
		if m[2*r*rows+r] != 1 {
			d := m[2*r*rows+r]
			scale := inverseTbl[d]
			for c := 0; c < columns; c++ {
				m[2*r*rows+c] = gfMul(m[2*r*rows+c], scale)
			}
		}
		for rowBelow := r + 1; rowBelow < rows; rowBelow++ {
			if m[2*rowBelow*rows+r] != 0 {
				scale := m[2*rowBelow*rows+r]
				for c := 0; c < columns; c++ {
					m[2*rowBelow*rows+c] ^= gfMul(scale, m[2*r*rows+c])
				}
			}
		}
	}
	for d := 0; d < rows; d++ {
		for rowAbove := 0; rowAbove < d; rowAbove++ {
			if m[2*rowAbove*rows+d] != 0 {
				scale := m[2*rowAbove*rows+d]
				for c := 0; c < columns; c++ {
					m[2*rowAbove*rows+c] ^= gfMul(scale, m[2*d*rows+c])
				}
			}
		}
	}
	return nil
}

func (m matrix) mul(right matrix, rows, cols int) matrix {
	ret := newMatrix(rows, cols)
	for i := 0; i < rows; i++ {
		for j := 0; j < cols; j++ {
			var v byte
			for k := 0; k < cols; k++ {
				v ^= gfMul(m[i*cols+k], right[k*cols+j])
			}
			ret[i*cols+j] = v
		}
	}
	return ret
}

func (m matrix) subMatrix(size int) matrix {
	ret := newMatrix(size, size)
	for i := 0; i < size; i++ {
		copy(ret[i*size:i*size+size], m[2*i*size+size:2*i*size+2*size])
	}
	return ret
}<commit_msg>fmt matrix<commit_after>package reedsolomon

import "errors"

type matrix []byte

func newMatrix(rows, cols int) matrix {
	m := make([]byte, rows*cols)
	return m
}

func genEncMatrixCauchy(data, parity int) matrix {
	rows := data + parity
	cols := data
	m := newMatrix(rows, cols)
	for i := 0; i < cols; i++ {
		m[i*data+i] = byte(1)
	}

	p := data * data
	for i := cols; i < rows; i++ {
		for j := 0; j < cols; j++ {
			d := i ^ j
			a := inverseTbl[d]
			m[p] = byte(a)
			p++
		}
	}
	return m
}

func genVandMatrix(rows, cols int) matrix {
	m := newMatrix(rows, cols)
	for i := 0; i < rows; i++ {
		for j := 0; j < cols; j++ {
			m[i*cols+j] = gfExp(byte(i), j)
		}
	}
	return m
}

func gfExp(b byte, n int) byte {
	if n == 0 {
		return 1
	}
	if b == 0 {
		return 0
	}
	a := logTbl[b]
	ret := int(a) * n
	for ret >= 255 {
		ret -= 255
	}
	return byte(expTbl[ret])
}

func genEncMatrixVand(data, parity int) (matrix, error) {
	total := data + parity
	vm := genVandMatrix(total, data)
	top := newMatrix(data, data)
	copy(top, vm[:data*data])
	topI, err := top.invert(data)
	if err != nil {
		return nil, err
	}
	return vm.mul(topI, total, data), nil
}

func (m matrix) invert(n int) (matrix, error) {
	raw := newMatrix(n, 2*n)
	// [m] -> [m|I]
	for i := 0; i < n; i++ {
		t := i * n
		copy(raw[2*t:2*t+n], m[t:t+n])
		raw[2*t+i+n] = byte(1)
	}
	// [m|I] -> [I|m']
	err := raw.gauss(n, 2*n)
	if err != nil {
		return nil, err
	}
	// [I|m'] -> [m']
	return raw.subMatrix(n), nil
}

func (m matrix) swap(i, j, n int) {
	for k := 0; k < n; k++ {
		m[i*n+k], m[j*n+k] = m[j*n+k], m[i*n+k]
	}
}

func gfMul(a, b byte) byte {
	return mulTbl[a][b]
}

var errSingular = errors.New("rs.invert: matrix is singular")

func (m matrix) gauss(rows, cols int) error {
	for i := 0; i < rows; i++ {
		if m[i*cols+i] == 0 {
			for j := i + 1; j < rows; j++ {
				if m[j*cols+i] != 0 {
					m.swap(i, j, cols)
					break
				}
			}
		}
		if m[i*cols+i] == 0 {
			return errSingular
		}
		if m[i*cols+i] != 1 {
			d := m[i*cols+i]
			scale := inverseTbl[d]
			for c := 0; c < cols; c++ {
				m[i*cols+c] = gfMul(m[i*cols+c], scale)
			}
		}
		for j := i + 1; j < rows; j++ {
			if m[j*cols+i] != 0 {
				scale := m[j*cols+i]
				for c := 0; c < cols; c++ {
					m[j*cols+c] ^= gfMul(scale, m[i*cols+c])
				}
			}
		}
	}
	for k := 0; k < rows; k++ {
		for j := 0; j < k; j++ {
			if m[j*cols+k] != 0 {
				scale := m[j*cols+k]
				for c := 0; c < cols; c++ {
					m[j*cols+c] ^= gfMul(scale, m[k*cols+c])
				}
			}
		}
	}
	return nil
}

func (m matrix) mul(right matrix, rows, cols int) matrix {
	r := newMatrix(rows, cols)
	for i := 0; i < rows; i++ {
		for j := 0; j < cols; j++ {
			var v byte
			for k := 0; k < cols; k++ {
				v ^= gfMul(m[i*cols+k], right[k*cols+j])
			}
			r[i*cols+j] = v
		}
	}
	return r
}

func (m matrix) subMatrix(n int) matrix {
	r := newMatrix(n, n)
	for i := 0; i < n; i++ {
		off := i * n
		copy(r[off:off+n], m[2*off+n:2*(off+n)])
	}
	return r
}
{'fraction_non_alphanumeric': 0.21151337247227658, 'fraction_numerical': 0.01679712981082844, 'mean_word_length': 2.71922377198302, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3330777', 'n_tokens_mistral': 3254, 'n_tokens_neox': 2917, 'n_words': 758}