<commit_before>package writer // import "a4.io/blobstash/pkg/filetree/writer"

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/dchest/blake2b"
	"github.com/restic/chunker"

	"a4.io/blobstash/pkg/filetree/filetreeutil/meta"
	"a4.io/blobstash/pkg/hashutil"
)

var (
	pol = chunker.Pol(0x3c657535c4d6f5)
)

func (up *Uploader) writeReader(f io.Reader, meta *meta.Meta) error { // (*WriteResult, error) {
	// writeResult := NewWriteResult()
	// Init the rolling checksum

	// reuse this buffer
	buf := make([]byte, 8*1024*1024)
	// Prepare the reader to compute the hash on the fly
	fullHash := blake2b.New256()
	freader := io.TeeReader(f, fullHash)
	chunkSplitter := chunker.New(freader, pol)
	// TODO don't read one byte at a time if meta.Size < chunker.ChunkMinSize
	// Prepare the blob writer
	var size uint
	for {
		chunk, err := chunkSplitter.Next(buf)
		if err == io.EOF {
			break
		}
		chunkHash := hashutil.Compute(chunk.Data)
		size += chunk.Length

		exists, err := up.bs.Stat(chunkHash)
		if err != nil {
			panic(fmt.Sprintf("DB error: %v", err))
		}
		if !exists {
			if err := up.bs.Put(chunkHash, chunk.Data); err != nil {
				panic(fmt.Errorf("failed to PUT blob %v", err))
			}
		}

		// Save the location and the blob hash into a sorted list (with the offset as index)
		meta.AddIndexedRef(int(size), chunkHash)

		if err != nil {
			return err
		}
	}
	meta.Size = int(size)
	return nil
	// writeResult.Hash = fmt.Sprintf("%x", fullHash.Sum(nil))
	// if writeResult.BlobsUploaded > 0 {
	// 	writeResult.FilesCount++
	// 	writeResult.FilesUploaded++
	// }<commit_msg>filetree/writer: store the blake2b hash of the file in data<commit_after>package writer // import "a4.io/blobstash/pkg/filetree/writer"

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/dchest/blake2b"
	"github.com/restic/chunker"

	"a4.io/blobstash/pkg/filetree/filetreeutil/meta"
	"a4.io/blobstash/pkg/hashutil"
)

var (
	pol = chunker.Pol(0x3c657535c4d6f5)
)

func (up *Uploader) writeReader(f io.Reader, meta *meta.Meta) error { // (*WriteResult, error) {
	// writeResult := NewWriteResult()
	// Init the rolling checksum

	// reuse this buffer
	buf := make([]byte, 8*1024*1024)
	// Prepare the reader to compute the hash on the fly
	fullHash := blake2b.New256()
	freader := io.TeeReader(f, fullHash)
	chunkSplitter := chunker.New(freader, pol)
	// TODO don't read one byte at a time if meta.Size < chunker.ChunkMinSize
	// Prepare the blob writer
	var size uint
	for {
		chunk, err := chunkSplitter.Next(buf)
		if err == io.EOF {
			break
		}
		chunkHash := hashutil.Compute(chunk.Data)
		size += chunk.Length

		exists, err := up.bs.Stat(chunkHash)
		if err != nil {
			panic(fmt.Sprintf("DB error: %v", err))
		}
		if !exists {
			if err := up.bs.Put(chunkHash, chunk.Data); err != nil {
				panic(fmt.Errorf("failed to PUT blob %v", err))
			}
		}

		// Save the location and the blob hash into a sorted list (with the offset as index)
		meta.AddIndexedRef(int(size), chunkHash)

		if err != nil {
			return err
		}
	}
	meta.Size = int(size)
	blake2bHash := fmt.Sprintf("%x", fullHash.Sum(nil))
	if meta.Data == nil {
		meta.Data = map[string]interface{}{
			"blake2b-hash": blake2bHash,
		}
	} else {
		meta.Data["blake2b-hash"] = blake2bHash
	}
	return nil
	// writeResult.Hash = fmt.Sprintf("%x", fullHash.Sum(nil))
	// if writeResult.BlobsUploaded > 0 {
	// 	writeResult.FilesCount++
	// 	writeResult.FilesUploaded++
	// }
{'fraction_non_alphanumeric': 0.14856646394439618, 'fraction_numerical': 0.01853460758760498, 'mean_word_length': 3.4225352112676055, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6014502', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1314, 'n_words': 314}