func WriteAndClose(r io.Reader, f *os.File, t CompressionType, hash string, size int64) (int64, error) {
	var err error
	defer f.Close()
	if size <= 0 {
		return -1, fmt.Errorf("invalid file size: %d", size)
	}
	chunkSize := uint32(defaultChunkSize)
	numChunks := int64(1)
	remainder := int64(0)
	if t == Zstandard {
		numChunks = size / int64(chunkSize)
		remainder = size % int64(chunkSize)
		if remainder > 0 {
			numChunks++
		}
	}
	numOffsets := numChunks + 1
	h := header{
		uncompressedSize: size,
		compression:      t,
		chunkSize:        chunkSize,
		chunkOffsets:     make([]int64, numOffsets),
	}
	h.chunkOffsets[0] = chunkTableOffset
	err = h.write(f)
	if err != nil {
		return -1, err
	}
	fileOffset := h.size()
	var n int64
	if t == Identity {
		hasher := sha256.New()
		n, err = io.Copy(io.MultiWriter(f, hasher), r)
		if err != nil {
			return -1, err
		}
		if n != size {
			return -1, fmt.Errorf("expected to copy %d bytes, actually copied %d bytes",
				size, n)
		}
		actualHash := hex.EncodeToString(hasher.Sum(nil))
		if actualHash != hash {
			return -1,
				fmt.Errorf("checksums don't match. Expected %s, found %s",
					hash, actualHash)
		}
		return n + fileOffset, f.Close()
	}
	nextChunk := 0  
	remainingRawData := size
	uncompressedChunk := make([]byte, chunkSize)
	hasher := sha256.New()
	for nextChunk < len(h.chunkOffsets)-1 {
		h.chunkOffsets[nextChunk] = fileOffset
		nextChunk++
		chunkEnd := int64(chunkSize)
		if remainingRawData <= int64(chunkSize) {
			chunkEnd = remainingRawData
		}
		remainingRawData -= chunkEnd
		_, err = io.ReadFull(r, uncompressedChunk[0:chunkEnd])
		if err != nil {
			return -1, err
		}
		compressedChunk := encoder.EncodeAll(uncompressedChunk[0:chunkEnd], nil)
		hasher.Write(uncompressedChunk[0:chunkEnd])
		written, err := f.Write(compressedChunk)
		if err != nil {
			return -1, err
		}
		fileOffset += int64(written)
	}
	h.chunkOffsets[nextChunk] = fileOffset
	bytesAfter, err := io.ReadFull(r, uncompressedChunk)
	if err == nil {
		return -1, fmt.Errorf("expected %d bytes but got at least %d more", size, bytesAfter)
	} else if err != io.EOF {
		return -1, err
	}
	actualHash := hex.EncodeToString(hasher.Sum(nil))
	if actualHash != hash {
		return -1, fmt.Errorf("checksums don't match. Expected %s, found %s",
			hash, actualHash)
	}
	_, err = f.Seek(chunkTableOffset, io.SeekStart)
	if err != nil {
		return -1, err
	}
	err = binary.Write(f, binary.LittleEndian, h.chunkOffsets)
	if err != nil {
		return -1, err
	}
	err = f.Sync()
	if err != nil {
		return -1, err
	}
	return fileOffset, f.Close()
}
Read from r and write to f, using CompressionType t.
Return the size on disk or an error if something went wrong.
package casblob
import (
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"github.com/buchgr/bazel-remote/utils/zstdpool"
	"github.com/klauspost/compress/zstd"
	syncpool "github.com/mostynb/zstdpool-syncpool"
)
type CompressionType uint8
const (
	Identity  CompressionType = 0
	Zstandard CompressionType = 1
)
var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel)  
var decoder, _ = zstd.NewReader(nil)                    
var encoderPool = zstdpool.GetEncoderPool()
var decoderPool = zstdpool.GetDecoderPool()
var errDecoderPoolFail error = errors.New("failed to get DecoderWrapper from pool")
var errEncoderPoolFail error = errors.New("failed to get EncoderWrapper from pool")
const defaultChunkSize = 1024 * 1024 * 1  
const skippableFrameMagicNumber = 0x184D2A50
type header struct {
	uncompressedSize int64            
	compression      CompressionType  
	chunkSize        uint32           
	chunkOffsets []int64
}
const chunkTableOffset = 4 + 4 + 8 + 1 + 4 + 8
func (h *header) size() int64 {
	return chunkTableOffset + (int64(len(h.chunkOffsets)) * 8)
}
func (h *header) frameSize() uint32 {
	return chunkTableOffset + (uint32(len(h.chunkOffsets)) * 8) - 4 - 4
}
type readCloserWrapper struct {
	*header
	rdr io.Reader  
	decoder *syncpool.DecoderWrapper  
	file    *os.File
}
var errWrongMagicNum = errors.New("expected magic number not found")
func readHeader(f *os.File) (*header, error) {
	var err error
	var h header
	fileInfo, err := f.Stat()
	if err != nil {
		return nil, err
	}
	foundFileSize := fileInfo.Size()
	if foundFileSize <= (chunkTableOffset + 16) {
		return nil, fmt.Errorf("file too small (%d) than the minimum header size (%d)",
			foundFileSize, (chunkTableOffset + 16))
	}
	var magicNumber uint32
	err = binary.Read(f, binary.LittleEndian, &magicNumber)
	if err != nil {
		return nil, fmt.Errorf("unable to read magic number: %w", err)
	}
	if magicNumber != skippableFrameMagicNumber {
		return nil, errWrongMagicNum
	}
	var frameSize uint32
	err = binary.Read(f, binary.LittleEndian, &frameSize)
	if err != nil {
		return nil, fmt.Errorf("unable to read frameSize: %w", err)
	}
	err = binary.Read(f, binary.LittleEndian, &h.uncompressedSize)
	if err != nil {
		return nil, err
	}
	err = binary.Read(f, binary.LittleEndian, &h.compression)
	if err != nil {
		return nil, err
	}
	err = binary.Read(f, binary.LittleEndian, &h.chunkSize)
	if err != nil {
		return nil, err
	}
	var numOffsets int64
	err = binary.Read(f, binary.LittleEndian, &numOffsets)
	if err != nil {
		return nil, err
	}
	if numOffsets < 2 {
		return nil, fmt.Errorf("internal error: need at least one chunk, found %d", numOffsets-1)
	}
	metadataSize := numOffsets*8 + 8 + 1 + 4 + 8
	if int64(frameSize) != metadataSize {
		return nil, fmt.Errorf("metadata frame size %d, but metadata size %d",
			frameSize, metadataSize)
	}
	h.chunkOffsets = make([]int64, numOffsets)
	err = binary.Read(f, binary.LittleEndian, h.chunkOffsets)
	if err != nil {
		return nil, err
	}
	prevOffset := int64(-1)
	for i := 0; int64(i) < numOffsets; i++ {
		if h.chunkOffsets[i] <= prevOffset {
			return nil,
				fmt.Errorf("offset table values should increase: %d -> %d",
					h.chunkOffsets[i], prevOffset)
		}
		prevOffset = h.chunkOffsets[i]
	}
	if prevOffset != foundFileSize {
		return nil,
			fmt.Errorf("final offset in chunk table %d should be file size %d",
				prevOffset, foundFileSize)
	}
	return &h, nil
}
func ExtractLogicalSize(rc io.ReadCloser) (io.ReadCloser, int64, error) {
	interesting := 16
	earlyHeader := make([]byte, interesting)
	n, err := io.ReadFull(rc, earlyHeader)
	if err != nil {
		return nil, -1, err
	}
	if n != 16 {
		return nil, -1, fmt.Errorf("Tried to read 16 header bytes, only read %d", n)
	}
	var uncompressedSize int64
	br := bytes.NewReader(earlyHeader[8:])
	err = binary.Read(br, binary.LittleEndian, &uncompressedSize)
	if err != nil {
		return nil, -1, err
	}
	if uncompressedSize <= 0 {
		return nil, -1, fmt.Errorf("Expected blob to have positive size, found %d",
			uncompressedSize)
	}
	return &multiReadCloser{
		Reader: io.MultiReader(bytes.NewReader(earlyHeader), rc),
		rc:     rc,
	}, uncompressedSize, nil
}
type multiReadCloser struct {
	io.Reader  
	rc        io.ReadCloser
}
func (m *multiReadCloser) Close() error {
	return m.rc.Close()
}
func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
	h, err := readHeader(f)
	if err != nil {
		f.Close()
		return nil, err
	}
	if expectedSize != -1 && h.uncompressedSize != expectedSize {
		f.Close()
		return nil, fmt.Errorf("expected a blob of size %d, found %d",
			expectedSize, h.uncompressedSize)
	}
	if h.compression == Identity {
		if offset > 0 {
			_, err = f.Seek(offset, io.SeekCurrent)
			if err != nil {
				f.Close()
				return nil, err
			}
		}
		return f, nil
	}
	if h.compression != Zstandard {
		f.Close()
		return nil,
			fmt.Errorf("internal error: unsupported compression type %d",
				h.compression)
	}
	chunkNum := int64(offset / int64(h.chunkSize))
	remainder := offset % int64(h.chunkSize)
	if chunkNum > 0 {
		_, err = f.Seek(h.chunkOffsets[chunkNum], io.SeekStart)
		if err != nil {
			f.Close()
			return nil, err
		}
	}
	if remainder == 0 {
		z, ok := decoderPool.Get().(*syncpool.DecoderWrapper)
		if !ok {
			f.Close()
			return nil, err
		}
		err := z.Reset(f)
		if err != nil {
			f.Close()
			return nil, err
		}
		return &readCloserWrapper{
			header:  h,
			rdr:     z,
			decoder: z,
			file:    f,
		}, nil
	}
	compressedFirstChunk := make([]byte, h.chunkOffsets[chunkNum+1]-h.chunkOffsets[chunkNum])
	_, err = io.ReadFull(f, compressedFirstChunk)
	if err != nil {
		f.Close()
		return nil, err
	}
	uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil)
	if err != nil {
		f.Close()
		return nil, err
	}
	if chunkNum == int64(len(h.chunkOffsets)-2) {
		r := bytes.NewReader(uncompressedFirstChunk[remainder:])
		f.Close()
		return ioutil.NopCloser(r), nil
	}
	z, ok := decoderPool.Get().(*syncpool.DecoderWrapper)
	if !ok {
		f.Close()
		return nil, errDecoderPoolFail
	}
	err = z.Reset(f)
	if err != nil {
		f.Close()
		decoderPool.Put(z)
		return nil, fmt.Errorf("Failed to setup zstd decoder: %w", err)
	}
	br := bytes.NewReader(uncompressedFirstChunk[remainder:])
	return &readCloserWrapper{
		header:  h,
		rdr:     io.MultiReader(br, z),
		decoder: z,
		file:    f,
	}, nil
}
func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
	h, err := readHeader(f)
	if err != nil {
		f.Close()
		return nil, err
	}
	if expectedSize != -1 && h.uncompressedSize != expectedSize {
		f.Close()
		return nil, fmt.Errorf("expected a blob of size %d, found %d",
			expectedSize, h.uncompressedSize)
	}
	if h.compression == Identity {
		if offset > 0 {
			_, err = f.Seek(offset, io.SeekCurrent)
			if err != nil {
				f.Close()
				return nil, err
			}
		}
		return GetLegacyZstdReadCloser(f)
	}
	if h.compression != Zstandard {
		f.Close()
		return nil, fmt.Errorf("unsupported compression type: %d",
			h.compression)
	}
	chunkNum := int64(offset / int64(h.chunkSize))
	remainder := offset % int64(h.chunkSize)
	if chunkNum > 0 {
		_, err = f.Seek(h.chunkOffsets[chunkNum], io.SeekStart)
		if err != nil {
			return nil, err
		}
	}
	if remainder == 0 {
		return f, nil
	}
	compressedFirstChunk := make([]byte, h.chunkOffsets[chunkNum+1]-h.chunkOffsets[chunkNum])
	_, err = io.ReadFull(f, compressedFirstChunk)
	if err != nil {
		f.Close()
		return nil, err
	}
	uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil)
	if err != nil {
		f.Close()
		return nil, err
	}
	chunkToRecompress := uncompressedFirstChunk[remainder:]
	recompressedChunk := encoder.EncodeAll(chunkToRecompress, nil)
	br := bytes.NewReader(recompressedChunk)
	if chunkNum == int64(len(h.chunkOffsets)-2) {
		f.Close()
		return ioutil.NopCloser(br), nil
	}
	return &readCloserWrapper{
		header: h,
		rdr:    io.MultiReader(br, f),
		file:   f,
	}, nil
}
func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) {
	enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper)
	if !ok {
		f.Close()
		return nil, errEncoderPoolFail
	}
	pr, pw := io.Pipe()
	enc.Reset(pw)
	go func() {
		_, err := enc.ReadFrom(f)
		if err != nil {
			log.Println("Error while reading/compressing file:", err)
			_ = pw.CloseWithError(err)
		}
		encoderPool.Put(enc)
		f.Close()
	}()
	return pr, nil
}
func (h *header) write(f *os.File) error {
	var err error
	err = binary.Write(f, binary.LittleEndian, uint32(skippableFrameMagicNumber))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.LittleEndian, h.frameSize())
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.LittleEndian, h.uncompressedSize)
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.LittleEndian, h.compression)
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.LittleEndian, h.chunkSize)
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.LittleEndian, int64(len(h.chunkOffsets)))
	if err != nil {
		return err
	}
	return binary.Write(f, binary.LittleEndian, h.chunkOffsets)
}
func (b *readCloserWrapper) Read(p []byte) (int, error) {
	return b.rdr.Read(p)
}
func (b *readCloserWrapper) Close() error {
	if b.decoder != nil {
		b.decoder.Close()
		b.decoder = nil
	}
	if b.file == nil {
		return nil
	}
	f := b.file
	b.file = nil
	return f.Close()
}