<commit_before>package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"io/ioutil"
)

const (
	kMaxEntryNumberInFile = 1024
)

type NodeInfo struct {
}

type Entry struct {
	Id       int
	ParentId int
	Dir      bool
	Key      string
	Name     string
	FsName   string
	fs       *FileSystem
	nodefs.Node
}

func NewDir(id int, name string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, "", "", parent, true)
}

func NewFile(id int, name string, fsName string, key string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, fsName, key, parent, false)
}

func newEntry(id int, name string, fsName string, key string, parent *Entry, dir bool) (*Entry, error) {
	if parent != nil && parent.Dir == false {
		return nil, errors.New("parent entry can't be file type")
	}
	entry := Entry{id, 0, dir, key, name, fsName, nil, nil}

	entry.Id = id
	if parent == nil {
		entry.ParentId = 0
	} else {
		entry.ParentId = parent.Id
	}
	entry.Key = key
	entry.Dir = dir
	entry.Name = name
	entry.FsName = fsName
	return &entry, nil
}

func ReadEntries(reader io.Reader, buff []*Entry) error {
	dec := gob.NewDecoder(reader)
	return dec.Decode(&buff)
}

func WriteEntries(writer io.Writer, buff []*Entry) error {
	enc := gob.NewEncoder(writer)
	return enc.Encode(buff)
}

func (e *Entry) Print() {
	if e.Dir {
		fmt.Printf("[Dir]: Id: %d, ParentId: %d, Name: %s, FsName: %s\n",
			e.Id, e.ParentId, e.Name, e.FsName)
	} else {
		fmt.Printf("Id: %d, ParentId: %d, Name: %s, FsName: %s, Key %s\n",
			e.Id, e.ParentId, e.Name, e.FsName, e.Key)
	}
}

func (e *Entry) Stat(out *fuse.Attr) {
	out.Mode = fuse.S_IFREG | 0444
	out.Size = e.fs.oss.Size(e.FsName)
}

func (e *Entry) Data() (data []byte) {
	fp, err := e.fs.oss.Open(e.FsName)
	if err != nil {
		return nil
	}
	fpd := MakeDecryptor(e.fs.config.Enc, fp)
	data, _ = ioutil.ReadAll(fpd)
	fpd.Close()
	fp.Close()
	return
}

func (n *Entry) OnMount(c *nodefs.FileSystemConnector) {
	n.fs.onMount()
}

func (n *Entry) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
	children := n.Inode().Children()
	stream = make([]fuse.DirEntry, 0, len(children))
	for k, v := range children {
		mode := fuse.S_IFREG | 0666
		if v.IsDir() {
			mode = fuse.S_IFDIR | 0777
		}
		stream = append(stream, fuse.DirEntry{
			Name: k,<commit_msg>fix data size issue<commit_after>package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"io/ioutil"
)

const (
	kMaxEntryNumberInFile = 1024
)

type Entry struct {
	Id       int
	ParentId int
	Dir      bool
	Key      string
	Name     string
	FsName   string
	fs       *FileSystem
	nodefs.Node
	size uint64
}

func NewDir(id int, name string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, "", "", parent, true)
}

func NewFile(id int, name string, fsName string, key string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, fsName, key, parent, false)
}

func newEntry(id int, name string, fsName string, key string, parent *Entry, dir bool) (*Entry, error) {
	if parent != nil && parent.Dir == false {
		return nil, errors.New("parent entry can't be file type")
	}
	entry := Entry{id, 0, dir, key, name, fsName, nil, nil, 0}

	entry.Id = id
	if parent == nil {
		entry.ParentId = 0
	} else {
		entry.ParentId = parent.Id
	}
	entry.Key = key
	entry.Dir = dir
	entry.Name = name
	entry.FsName = fsName
	return &entry, nil
}

func ReadEntries(reader io.Reader, buff []*Entry) error {
	dec := gob.NewDecoder(reader)
	return dec.Decode(&buff)
}

func WriteEntries(writer io.Writer, buff []*Entry) error {
	enc := gob.NewEncoder(writer)
	return enc.Encode(buff)
}

func (e *Entry) Print() {
	if e.Dir {
		fmt.Printf("[Dir]: Id: %d, ParentId: %d, Name: %s, FsName: %s\n",
			e.Id, e.ParentId, e.Name, e.FsName)
	} else {
		fmt.Printf("Id: %d, ParentId: %d, Name: %s, FsName: %s, Key %s\n",
			e.Id, e.ParentId, e.Name, e.FsName, e.Key)
	}
}

func (e *Entry) Stat(out *fuse.Attr) {
	out.Mode = fuse.S_IFREG | 0444
	if e.size != 0 {
		out.Size = e.fs.oss.Size(e.FsName)
		e.size = out.Size
	} else {
		out.Size = e.size
	}
}

func (e *Entry) Data() (data []byte) {
	fp, err := e.fs.oss.Open(e.FsName)
	if err != nil {
		return nil
	}
	fpd := MakeDecryptor(e.fs.config.Enc, fp)
	data, _ = ioutil.ReadAll(fpd)

	out := fuse.Attr{}
	e.Stat(&out)
	size := int(out.Size)
	if len(data) < size {
		data = append(data, make([]byte, size-len(data))...)
	}

	fpd.Close()
	fp.Close()
	return
}

func (n *Entry) OnMount(c *nodefs.FileSystemConnector) {
	n.fs.onMount()
}

func (n *Entry) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
	children := n.Inode().Children()
	stream = make([]fuse.DirEntry, 0, len(children))
	for k, v := range children {
		mode := fuse.S_IFREG | 0666
		if v.IsDir() {
			mode = fuse.S_IFDIR | 0777
		}
		stream = append(stream, fuse.DirEntry{
			Name: k,
{'fraction_non_alphanumeric': 0.1631916599839615, 'fraction_numerical': 0.008420208500400962, 'mean_word_length': 3.462432915921288, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7329269', 'n_tokens_mistral': 2117, 'n_tokens_neox': 2021, 'n_words': 526}