func ValidateDataFiles(t testing.TB, expectedDir, actualDir string) bool {
	type indexEntry struct {
		hash         [16]byte
		found, equal bool
	}
	outcome := true
	expectedFS := afero.NewBasePathFs(afero.NewOsFs(), expectedDir)
	actualFS := afero.NewBasePathFs(afero.NewOsFs(), expectedDir)
	expected := iradix.New()
	require.NoError(t, afero.Walk(expectedFS, "/", func(pth string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return nil
		}
		rpth, _ := expectedFS.(*afero.BasePathFs).RealPath(pth)
		data, err := ioutil.ReadFile(rpth)
		require.NoError(t, err)
		expected, _, _ = expected.Insert([]byte(pth), &indexEntry{
			hash: md5.Sum(data),
		})
		return nil
	}))
	actual := iradix.New()
	require.NoError(t, afero.Walk(actualFS, "/", func(pth string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() || model.IsGeneratedFile(pth) {
			return nil
		}
		rpth, _ := expectedFS.(*afero.BasePathFs).RealPath(pth)
		data, err := ioutil.ReadFile(rpth)
		require.NoError(t, err)
		hash := md5.Sum(data)  
		obj, found := expected.Get([]byte(pth))
		var reference *indexEntry
		var equal bool
		if found {
			reference = obj.(*indexEntry)
			reference.found = found
			equal = hash == reference.hash
		}
		actual, _, _ = actual.Insert([]byte(pth), &indexEntry{
			hash:  hash,  
			found: found,
			equal: equal,
		})
		return nil
	}))
	it := expected.Root().Iterator()
	for k, obj, ok := it.Next(); ok; k, obj, ok = it.Next() {
		reference := obj.(*indexEntry)
		if !assert.Truef(t, reference.found, "file %s in reference not found in destination", string(k)) {
			outcome = false
		}
	}
	it = actual.Root().Iterator()
	for k, obj, ok := it.Next(); ok; k, obj, ok = it.Next() {
		data := obj.(*indexEntry)
		if !assert.Truef(t, data.found, "file %s in destination not found in reference", string(k)) {
			outcome = false
		}
		if !assert.Truef(t, data.equal, "file %s differ between destination and reference", string(k)) {
			outcome = false
		}
	}
	return outcome
}
ValidateDataFiles is a folder differ
package mocks
import (
	"bytes"
	"context"
	"crypto/md5"  
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"
	iradix "github.com/hashicorp/go-immutable-radix"
	"github.com/oneconcern/datamon/pkg/cafs"
	context2 "github.com/oneconcern/datamon/pkg/context"
	"github.com/oneconcern/datamon/pkg/model"
	"github.com/oneconcern/datamon/pkg/storage"
	"github.com/oneconcern/datamon/pkg/storage/localfs"
	"github.com/segmentio/ksuid"
	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
	"gopkg.in/yaml.v2"
)
type TestEnv struct {
	BundleID               string
	LeafSize               uint32
	ReBundleEntriesPerFile int
	Repo string
	TestRoot  string
	SourceDir string
	BlobDir  string
	MetaDir  string
	VmetaDir string
	Wal      string
	ReadLog  string
	DestinationDir   string
	Original         string
	DataDir          string
	ReArchiveMetaDir string
	ReArchiveBlobDir string
	PathToMount   string
	PathToStaging string
}
var (
	onceTimestamp sync.Once
	timestamp     time.Time
)
func GetTestTimeStamp() time.Time {
	onceTimestamp.Do(func() {
		timestamp = model.GetBundleTimeStamp()
	})
	t := timestamp
	return t
}
func TestLogger() *zap.Logger {
	if os.Getenv("DEBUG_TEST") != "" {
		l, _ := zap.NewDevelopment()  
		return l
	}
	return zap.NewNop()  
}
func FakeContext(meta, blob string) context2.Stores {
	var metaStore, blobStore storage.Store
	if meta != "" {
		metaStore = localfs.New(afero.NewBasePathFs(afero.NewOsFs(), meta))
	}
	if blob != "" {
		blobStore = localfs.New(afero.NewBasePathFs(afero.NewOsFs(), blob))
	}
	ctx := context2.New()
	ctx.SetMetadata(metaStore)
	ctx.SetBlob(blobStore)
	return ctx
}
func FakeContext2(meta, vmeta, blob string) context2.Stores {
	var vmetaStore storage.Store
	ctx := FakeContext(meta, blob)
	if vmeta != "" {
		vmetaStore = localfs.New(afero.NewBasePathFs(afero.NewOsFs(), vmeta), localfs.WithLock(true))
	}
	ctx.SetVMetadata(vmetaStore)
	return ctx
}
func FakeRepoDescriptor(name string) model.RepoDescriptor {
	return model.RepoDescriptor{
		Name:        name,
		Description: "test",
		Timestamp:   time.Time{},
		Contributor: model.Contributor{
			Name:  "test",
			Email: "t@test.com",
		},
	}
}
func FakeBundleDescriptor(bundleEntriesFileCount uint64, ev TestEnv) model.BundleDescriptor {
	return model.BundleDescriptor{
		ID:                     ev.BundleID,
		LeafSize:               ev.LeafSize,
		Message:                "test bundle",
		Timestamp:              GetTestTimeStamp(),
		Contributors:           []model.Contributor{{Name: "dev", Email: "dev@dev.com"}},
		BundleEntriesFileCount: bundleEntriesFileCount,
	}
}
func ValidatePublish(t testing.TB, store storage.Store, bundleEntriesFileCount uint64, ev TestEnv) {
	bundleDescriptor := ReadBundleDescriptor(t, store, model.GetConsumablePathToBundle(ev.BundleID))
	require.Equal(t, bundleDescriptor, FakeBundleDescriptor(bundleEntriesFileCount, ev))
	ValidateDataFiles(t, ev.Original, filepath.Join(ev.DestinationDir, ev.DataDir))
}
func ReadBundleDescriptor(t testing.TB, store storage.Store, pathToBundle string) model.BundleDescriptor {
	var bundleDescriptor model.BundleDescriptor
	reader, err := store.Get(context.Background(), pathToBundle)
	require.NoError(t, err)
	bundleDescriptorBuffer, err := ioutil.ReadAll(reader)
	require.NoError(t, err)
	err = yaml.Unmarshal(bundleDescriptorBuffer, &bundleDescriptor)
	require.NoError(t, err)
	return bundleDescriptor
}
func FakeDataFile(t testing.TB, store storage.Store, ev TestEnv) model.BundleEntry {
	ksuid, err := ksuid.NewRandom()
	require.NoError(t, err)
	var size = 2 * int(ev.LeafSize)
	file := filepath.Join(ev.Original, ksuid.String())
	require.NoError(t, cafs.GenerateFile(file, size, ev.LeafSize))
	fs, err := cafs.New(
		cafs.LeafSize(ev.LeafSize),
		cafs.Backend(store),
		cafs.Logger(TestLogger()),
	)
	require.NoError(t, err)
	keys, err := cafs.GenerateCAFSChunks(file, fs)
	require.NoError(t, err)
	return model.BundleEntry{
		Hash:         keys.String(),
		NameWithPath: filepath.Join(ev.DataDir, ksuid.String()),
		FileMode:     0700,
		Size:         uint64(size),
	}
}