<commit_before>package main

import (
	"bytes"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, blockgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.
// TODO: Seek.

type reader struct {
	rawR    io.ReadSeeker
	zipR    io.Reader
	index   []Record
	readBuf *bytes.Buffer
	trailer *Trailer
}

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	return offset, nil
}

// Return start (prev offset) and end (curr offset) of the block currOff is
// located in. If currOff is 0, the startoffset of the first and second record is
// returned. If currOff is at the end of file the end offset of the last block
// is returned twice.  The difference between prev record and curr block is then
// equal to 0.
func (r *reader) blockLookup(currOff int64) (*Record, *Record) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})

	// Beginning of the file, first block: prev offset is 0, curr offset is 1
	if i == 0 {
		return &r.index[i], &r.index[i+1]
	}

	// End of the file, last block: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}
	r.trailer = &Trailer{}
	r.trailer.unmarshal(buf[:])

	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err := r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		return err
	}

	trailerBuf := make([]byte, r.trailer.indexSize)
	if _, err := r.rawR.Read(trailerBuf); err != nil {
		return err
	}

	// Build index with Records. A record encapsulates a raw offset and the
	// compressed offset it is mapped to.
	prevRecord := Record{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexBlockSize); i++ {
		currRecord := Record{}
		currRecord.unmarshal(trailerBuf)

		if prevRecord.rawOff >= currRecord.rawOff && prevRecord.zipOff >= currRecord.zipOff {
			return ErrBadIndex
		}

		r.index = append(r.index, currRecord)
		trailerBuf = trailerBuf[IndexBlockSize:]
	}

	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	return nil
}

func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	read := 0
	for {
		if r.readBuf.Len() != 0 {
			n := copy(p, r.readBuf.Next(len(p)))
			read += n
			p = p[n:]
		}

		if len(p) == 0 {
			break
		}

		if _, err := r.readBlock(); err != nil {
			return read, err
		}
	}

	return read, nil
}

func (r *reader) readBlock() (int64, error) {
	// Get current position of reader (offset of the compressed file).
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get the start and end record of the block currOff is located in
	prevRecord, currRecord := r.blockLookup(currOff)
	if currRecord == nil || prevRecord == nil {
		return 0, ErrBadIndex
	}

	// Blocksize should only be 0 on empty file or at the end of file.
	blockSize := currRecord.rawOff - prevRecord.rawOff
	if blockSize == 0 {
		return 0, io.EOF
	}

	currZipOff := prevRecord.zipOff
	if _, err = r.rawR.Seek(currZipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	return io.CopyN(r.readBuf, r.zipR, blockSize)
}

// Return a new ReadSeeker with compression support. As random access is the
// purpose of this layer, a ReadSeeker is required as parameter. The used
// compression algorithm is chosen based on trailer information.
func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:    r,
		zipR:    snappy.NewReader(r),
		readBuf: &bytes.Buffer{},
	}
}<commit_msg>reader.go: Variable name cleanup + comments added.<commit_after>package main

import (
	"bytes"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, blockgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.
// TODO: Seek.

type reader struct {
	// Underlying raw, compressed datastream.
	rawR io.ReadSeeker

	// Decompression layer, reader is based on chosen algorithm.
	zipR io.Reader

	// Index with records which contain block offsets.
	index []Record

	// Buffer holds currently read data; MaxBlockSize.
	readBuf *bytes.Buffer

	// Structure with parsed trailer.
	trailer *Trailer
}

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	return offset, nil
}

// Return start (prev offset) and end (curr offset) of the block currOff is
// located in. If currOff is 0, the startoffset of the first and second record is
// returned. If currOff is at the end of file the end offset of the last block
// is returned twice.  The difference between prev record and curr block is then
// equal to 0.
func (r *reader) blockLookup(currOff int64) (*Record, *Record) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})

	// Beginning of the file, first block: prev offset is 0, curr offset is 1
	if i == 0 {
		return &r.index[i], &r.index[i+1]
	}

	// End of the file, last block: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	// Goto end of file and read trailer buffer.
	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}
	r.trailer = &Trailer{}
	r.trailer.unmarshal(buf[:])

	// Seek and read index into buffer.
	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err := r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		return err
	}
	indexBuf := make([]byte, r.trailer.indexSize)
	if _, err := r.rawR.Read(indexBuf); err != nil {
		return err
	}

	// Build index with Records. A record encapsulates a raw offset and the
	// compressed offset it is mapped to.
	prevRecord := Record{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexBlockSize); i++ {
		currRecord := Record{}
		currRecord.unmarshal(indexBuf)

		if prevRecord.rawOff >= currRecord.rawOff {
			return ErrBadIndex
		}

		if prevRecord.zipOff >= currRecord.zipOff {
			return ErrBadIndex
		}

		r.index = append(r.index, currRecord)
		indexBuf = indexBuf[IndexBlockSize:]
	}

	// Set reader to beginning of file
	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	return nil
}

// Read reads len(p) bytes from the compressed stream into p.
func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	read := 0
	for {
		if r.readBuf.Len() != 0 {
			n := copy(p, r.readBuf.Next(len(p)))
			read += n
			p = p[n:]
		}

		if len(p) == 0 {
			break
		}

		if _, err := r.readBlock(); err != nil {
			return read, err
		}
	}

	return read, nil
}

func (r *reader) readBlock() (int64, error) {
	// Get current position of the reader; offset of the compressed file.
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get the start and end record of the block currOff is located in.
	prevRecord, currRecord := r.blockLookup(currOff)
	if currRecord == nil || prevRecord == nil {
		return 0, ErrBadIndex
	}

	// Determinate uncompressed blocksize; should only be 0 on empty file or at the end of file.
	blockSize := currRecord.rawOff - prevRecord.rawOff
	if blockSize == 0 {
		return 0, io.EOF
	}

	// Set reader to compressed offset.
	if _, err = r.rawR.Seek(prevRecord.zipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	return io.CopyN(r.readBuf, r.zipR, blockSize)
}

// Return a new ReadSeeker with compression support. As random access is the
// purpose of this layer, a ReadSeeker is required as parameter. The used
// compression algorithm is chosen based on trailer information.
func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:    r,
		zipR:    snappy.NewReader(r),
		readBuf: &bytes.Buffer{},
	}
}
{'fraction_non_alphanumeric': 0.13370030581039755, 'fraction_numerical': 0.009541284403669725, 'mean_word_length': 3.2362694300518133, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7469041', 'n_tokens_mistral': 3160, 'n_tokens_neox': 2957, 'n_words': 971}