<commit_before>package main

import (
	"errors"
	"hash"
	"hash/crc32"
	"log"
	"os"
	"sort"
	"sync"
	"time"

	"bazil.org/fuse"

	pb "github.com/creiht/formic/proto"
	"github.com/gholt/brimtime"
	"github.com/gholt/store"
	"github.com/gogo/protobuf/proto"
	"github.com/pandemicsyn/oort/api"
	"github.com/spaolacci/murmur3"
	"google.golang.org/grpc"
)

const (
	InodeEntryVersion = 1
	DirEntryVersion   = 1
	FileBlockVersion  = 1
)

type FileService interface {
	GetAttr(id []byte) (*pb.Attr, error)
	SetAttr(id []byte, attr *pb.Attr, valid uint32) (*pb.Attr, error)
	Create(parent, id []byte, inode uint64, name string, attr *pb.Attr, isdir bool) (string, *pb.Attr, error)
	Update(id []byte, block, size, blocksize uint64, mtime int64) error
	Lookup(parent []byte, name string) (string, *pb.Attr, error)
	ReadDirAll(id []byte) (*pb.ReadDirAllResponse, error)
	Remove(parent []byte, name string) (int32, error)
	Symlink(parent, id []byte, name string, target string, attr *pb.Attr, inode uint64) (*pb.SymlinkResponse, error)
	Readlink(id []byte) (*pb.ReadlinkResponse, error)
	Getxattr(id []byte, name string) (*pb.GetxattrResponse, error)
	Setxattr(id []byte, name string, value []byte) (*pb.SetxattrResponse, error)
	Listxattr(id []byte) (*pb.ListxattrResponse, error)
	Removexattr(id []byte, name string) (*pb.RemovexattrResponse, error)
	Rename(oldParent, newParent []byte, oldName, newName string) (*pb.RenameResponse, error)
	GetChunk(id []byte) ([]byte, error)
	WriteChunk(id, data []byte) error
}

var ErrStoreHasNewerValue = errors.New("Error store already has newer value")

type OortFS struct {
	vaddr  string
	gaddr  string
	vstore store.ValueStore
	gstore api.GroupStore
	hasher func() hash.Hash32
	sync.RWMutex
}

func NewOortFS(vaddr, gaddr string, insecureSkipVerify bool, grpcOpts ...grpc.DialOption) (*OortFS, error) {
	// TODO: This all eventually needs to replaced with value and group rings
	var err error
	o := &OortFS{
		vaddr:  vaddr,
		gaddr:  gaddr,
		hasher: crc32.NewIEEE,
	}
	// TODO: These 10s here are the number of grpc streams the api can make per
	// request type; should likely be configurable somewhere along the line,
	// but hardcoded for now.
	o.vstore, err = api.NewValueStore(vaddr, 10, insecureSkipVerify, grpcOpts...)
	if err != nil {
		return &OortFS{}, err
	}
	o.gstore, err = api.NewGroupStore(gaddr, 10, insecureSkipVerify, grpcOpts...)
	if err != nil {
		return &OortFS{}, err
	}<commit_msg>Forgot you have to Startup a Store to use it<commit_after>package main

import (
	"errors"
	"hash"
	"hash/crc32"
	"log"
	"os"
	"sort"
	"sync"
	"time"

	"bazil.org/fuse"

	pb "github.com/creiht/formic/proto"
	"github.com/gholt/brimtime"
	"github.com/gholt/store"
	"github.com/gogo/protobuf/proto"
	"github.com/pandemicsyn/oort/api"
	"github.com/spaolacci/murmur3"
	"google.golang.org/grpc"
)

const (
	InodeEntryVersion = 1
	DirEntryVersion   = 1
	FileBlockVersion  = 1
)

type FileService interface {
	GetAttr(id []byte) (*pb.Attr, error)
	SetAttr(id []byte, attr *pb.Attr, valid uint32) (*pb.Attr, error)
	Create(parent, id []byte, inode uint64, name string, attr *pb.Attr, isdir bool) (string, *pb.Attr, error)
	Update(id []byte, block, size, blocksize uint64, mtime int64) error
	Lookup(parent []byte, name string) (string, *pb.Attr, error)
	ReadDirAll(id []byte) (*pb.ReadDirAllResponse, error)
	Remove(parent []byte, name string) (int32, error)
	Symlink(parent, id []byte, name string, target string, attr *pb.Attr, inode uint64) (*pb.SymlinkResponse, error)
	Readlink(id []byte) (*pb.ReadlinkResponse, error)
	Getxattr(id []byte, name string) (*pb.GetxattrResponse, error)
	Setxattr(id []byte, name string, value []byte) (*pb.SetxattrResponse, error)
	Listxattr(id []byte) (*pb.ListxattrResponse, error)
	Removexattr(id []byte, name string) (*pb.RemovexattrResponse, error)
	Rename(oldParent, newParent []byte, oldName, newName string) (*pb.RenameResponse, error)
	GetChunk(id []byte) ([]byte, error)
	WriteChunk(id, data []byte) error
}

var ErrStoreHasNewerValue = errors.New("Error store already has newer value")

type OortFS struct {
	vaddr  string
	gaddr  string
	vstore store.ValueStore
	gstore api.GroupStore
	hasher func() hash.Hash32
	sync.RWMutex
}

func NewOortFS(vaddr, gaddr string, insecureSkipVerify bool, grpcOpts ...grpc.DialOption) (*OortFS, error) {
	// TODO: This all eventually needs to replaced with value and group rings
	var err error
	o := &OortFS{
		vaddr:  vaddr,
		gaddr:  gaddr,
		hasher: crc32.NewIEEE,
	}
	// TODO: These 10s here are the number of grpc streams the api can make per
	// request type; should likely be configurable somewhere along the line,
	// but hardcoded for now.
	o.vstore, err = api.NewValueStore(vaddr, 10, insecureSkipVerify, grpcOpts...)
	if err != nil {
		return &OortFS{}, err
	}
    if err = o.vstore.Startup(); err != nil {
		return &OortFS{}, err
    }
	o.gstore, err = api.NewGroupStore(gaddr, 10, insecureSkipVerify, grpcOpts...)
	if err != nil {
		return &OortFS{}, err
	}
    if err = o.gstore.Startup(); err != nil {
		return &OortFS{}, err
    }
{'fraction_non_alphanumeric': 0.13694834751632692, 'fraction_numerical': 0.011082525232535128, 'mean_word_length': 4.578366445916115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '456081', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1868, 'n_words': 521}