<commit_before>/*
 * Copyright (c) 2013 Zhen, LLC. http://zhen.io. All rights reserved.
 * Use of this source code is governed by the Apache 2.0 license.
 *
 */

package variablebyte

import (
	"fmt"
	"errors"
	"github.com/reducedb/encoding"
	"github.com/reducedb/encoding/buffers"
)

type IntegratedVariableByte struct {

}

var _ encoding.Integer = (*IntegratedVariableByte)(nil)

func NewIntegratedVariableByte() encoding.Integer {
	return &IntegratedVariableByte{}
}

func (this *IntegratedVariableByte) Compress(in []uint32, inpos *encoding.Cursor, inlength int, out []uint32, outpos *encoding.Cursor) error {
	if inlength == 0 {
		return errors.New("variablebyte/Compress: inlength = 0. No work done.")
	}

	fmt.Printf("variablebyte/Compress: after inlength = %d\n", inlength)

	buf := buffers.NewByteBuffer(inlength*8)
	initoffset := uint32(0)

	for k := inpos.Get(); k < inpos.Get() + inlength; k++ {
		val := in[k] - initoffset
		fmt.Printf("variablebyte/Compress: val = %d, initoffset = %d\n", val, initoffset)
		initoffset = in[k]

		// This section emulates a do..while loop
		b := val & 127
		fmt.Printf("variablebyte/Compress: before val = %d, b = %d\n", val, b)
		val = val>>7

		if val != 0 {
			b |= 128
		}
		fmt.Printf("variablebyte/Compress: after val = %d, b = %d\n", val, b)

		buf.Put(byte(b))

		for val != 0 {
			b = val & 127
			fmt.Printf("variablebyte/Compress: before val = %d, b = %d\n", val, b)
			val = val>>7

			if val != 0 {
				b |= 128
			}
			fmt.Printf("variablebyte/Compress: after val = %d, b = %d\n", val, b)

			buf.Put(byte(b))
		}
	}

	for buf.Position()%4 != 0 {
		fmt.Printf("variablebyte/Compress: putting 128\n")
		buf.Put(128)
	}

	length := buf.Position()
	buf.Flip()
	ibuf := buf.AsUint32Buffer()
	fmt.Printf("variablebyte/Compress: l = %d, outpos = %d, ibuf = %v, buf = %v\n", length/4, outpos.Get(), ibuf, buf)
	err := ibuf.GetUint32s(out, outpos.Get(), length/4)
	if err != nil {
		fmt.Printf("variablebyte/Compress: error with GetUint32s - %v\n", err)
		return err
	}
	outpos.Add(length/4)
	inpos.Add(inlength)
	fmt.Printf("variablebyte/Compress: out = %v\n", out)

	return nil
}

func (this *IntegratedVariableByte) Uncompress(in []uint32, inpos *encoding.Cursor, inlength int, out []uint32, outpos *encoding.Cursor) error {
	if inlength == 0 {
		return errors.New("variablebyte/Uncompress: inlength = 0. No work done.")
	}

	//fmt.Printf("variablebyte/Uncompress: after inlength = %d\n", inlength)

	s := 0
	p := inpos.Get()
	finalp := inpos.Get() + inlength
	tmpoutpos := outpos.Get()
	initoffset := uint32(0)<commit_msg>Removed printf debug lines<commit_after>/*
 * Copyright (c) 2013 Zhen, LLC. http://zhen.io. All rights reserved.
 * Use of this source code is governed by the Apache 2.0 license.
 *
 */

package variablebyte

import (
	"errors"
	"github.com/reducedb/encoding"
	"github.com/reducedb/encoding/buffers"
)

type IntegratedVariableByte struct {

}

var _ encoding.Integer = (*IntegratedVariableByte)(nil)

func NewIntegratedVariableByte() encoding.Integer {
	return &IntegratedVariableByte{}
}

func (this *IntegratedVariableByte) Compress(in []uint32, inpos *encoding.Cursor, inlength int, out []uint32, outpos *encoding.Cursor) error {
	if inlength == 0 {
		return errors.New("variablebyte/Compress: inlength = 0. No work done.")
	}

	//fmt.Printf("variablebyte/Compress: after inlength = %d\n", inlength)

	buf := buffers.NewByteBuffer(inlength*8)
	initoffset := uint32(0)

	for k := inpos.Get(); k < inpos.Get() + inlength; k++ {
		val := in[k] - initoffset
		//fmt.Printf("variablebyte/Compress: val = %d, initoffset = %d\n", val, initoffset)
		initoffset = in[k]

		// This section emulates a do..while loop
		b := val & 127
		//fmt.Printf("variablebyte/Compress: before val = %d, b = %d\n", val, b)
		val = val>>7

		if val != 0 {
			b |= 128
		}
		//fmt.Printf("variablebyte/Compress: after val = %d, b = %d\n", val, b)

		buf.Put(byte(b))

		for val != 0 {
			b = val & 127
			//fmt.Printf("variablebyte/Compress: before val = %d, b = %d\n", val, b)
			val = val>>7

			if val != 0 {
				b |= 128
			}
			//fmt.Printf("variablebyte/Compress: after val = %d, b = %d\n", val, b)

			buf.Put(byte(b))
		}
	}

	for buf.Position()%4 != 0 {
		//fmt.Printf("variablebyte/Compress: putting 128\n")
		buf.Put(128)
	}

	length := buf.Position()
	buf.Flip()
	ibuf := buf.AsUint32Buffer()
	//fmt.Printf("variablebyte/Compress: l = %d, outpos = %d, ibuf = %v, buf = %v\n", length/4, outpos.Get(), ibuf, buf)
	err := ibuf.GetUint32s(out, outpos.Get(), length/4)
	if err != nil {
		//fmt.Printf("variablebyte/Compress: error with GetUint32s - %v\n", err)
		return err
	}
	outpos.Add(length/4)
	inpos.Add(inlength)
	//fmt.Printf("variablebyte/Compress: out = %v\n", out)

	return nil
}

func (this *IntegratedVariableByte) Uncompress(in []uint32, inpos *encoding.Cursor, inlength int, out []uint32, outpos *encoding.Cursor) error {
	if inlength == 0 {
		return errors.New("variablebyte/Uncompress: inlength = 0. No work done.")
	}

	//fmt.Printf("variablebyte/Uncompress: after inlength = %d\n", inlength)

	s := 0
	p := inpos.Get()
	finalp := inpos.Get() + inlength
	tmpoutpos := outpos.Get()
	initoffset := uint32(0)
{'fraction_non_alphanumeric': 0.16544543000385653, 'fraction_numerical': 0.023139220979560355, 'mean_word_length': 3.672972972972973, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3489924', 'n_tokens_mistral': 2098, 'n_tokens_neox': 1928, 'n_words': 487}