<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigint.ca/fuse"
)

const (
	tUnknown = iota
	tPrefix
	tObject
)

type s3fs struct {
	bucket string
	key    string
	kind   int
	size   uint64
	body   io.ReadCloser
}

func (ctx s3fs) String() string {
	var kind string
	switch ctx.kind {
	case tUnknown:
		kind = "unknown"
	case tPrefix:
		kind = "prefix"
	case tObject:
		kind = "object"
	}
	return fmt.Sprintf("{key: %#v, kind: %s}", ctx.key, kind)
}

func (ctx s3fs) Root() (fuse.Node, fuse.Error) {
	dprintf("Root: %v", ctx)
	ctx.key = ""
	ctx.kind = tPrefix
	return ctx, nil
}

func (ctx s3fs) Attr() fuse.Attr {
	dprintf("Attr: %v", ctx)
	switch ctx.kind {
	case tPrefix:
		dprintf("Attr: type: directory")
		return fuse.Attr{
			Mode: os.ModeDir | 0755,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	case tObject:
		dprintf("Attr: type: regular file")
		return fuse.Attr{
			Mode: 0644,
			Size: ctx.size,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	default:
		dprintf("Attr: type: unsupported")
		return fuse.Attr{}
	}
}

func (ctx s3fs) Lookup(name string, intr fuse.Intr) (fuse.Node, fuse.Error) {
	dprintf("Lookup: %#v (ctx.key: %#v)", name, ctx.key)
	key := ctx.key + "/" + name
	if ctx.key == "" {
		key = name
	}
	kind, obj, ok := s3Lookup(key)
	if !ok {
		dprintf("Lookup: returned ENOENT for %#v / %#v", ctx.key, name)
		return nil, fuse.ENOENT
	}
	ctx.key = key
	ctx.kind = kind
	if obj != nil {
		ctx.size = uint64(*obj.ContentLength)
		ctx.body = obj.Body
	} else {
		ctx.size = 0
		ctx.body = nil
	}
	dprintf("Lookup: got %v", ctx)
	return ctx, nil
}

func (ctx s3fs) ReadDir(intr fuse.Intr) ([]fuse.Dirent, fuse.Error) {
	dprintf("ReadDir: %v", ctx)
	if ctx.kind != tPrefix {
		return nil, fuse.EIO
	}

	objs, tPrefixes, err := s3ListDir(ctx.key)
	if err != nil {
		dprintf("ReadDir: %v", err)
	}

	out := make([]fuse.Dirent, 0, len(objs)+len(tPrefixes))
	for _, obj := range objs {
		out = append(out, fuse.Dirent{Name: filepath.Base(obj)})
	}
	for _, pre := range tPrefixes {
		out = append(out, fuse.Dirent{Name: filepath.Base(pre)})
	}

	dprintf("ReadDir: got %d dirents: %v", len(out), out)
	return out, nil
}

func (ctx s3fs) ReadAll(intr fuse.Intr) ([]byte, fuse.Error) {
	dprintf("ReadAll: %v", ctx)
	defer ctx.body.Close()
	buf, err := ioutil.ReadAll(ctx.body)
	if err != nil {
		dprintf("ReadAll: error reading %#v: %v", ctx.key, err)
		return nil, fuse.EIO
	}
	return buf, nil
}

func (ctx s3fs) Create(req *fuse.CreateRequest, res *fuse.CreateResponse, intr fuse.Intr) (fuse.Node, fuse.Handle, fuse.Error) {
	dprintf("Create: %v (ctx.key = %#v)", req, ctx.key)
	if ctx.key == "" {
		ctx.key = req.Name
	} else {
		ctx.key += "/" + req.Name
	}
	if req.Mode&os.ModeDir > 0 {
		ctx.kind = tPrefix
	} else {
		ctx.kind = tObject
	}
	return ctx, ctx, nil
}

func (ctx s3fs) Remove(req *fuse.RemoveRequest, intr fuse.Intr) fuse.Error {
	dprintf("Remove: %v (ctx.key = %#v)", req, ctx.key)
	key := ctx.key + "/" + req.Name
	err := s3RemoveObj(key)
	if err != nil {
		dprintf("Remove: error removing %#v: %v", key, err)
		return fuse.EIO<commit_msg>don't prepend slash when prefix is empty<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigint.ca/fuse"
)

const (
	tUnknown = iota
	tPrefix
	tObject
)

type s3fs struct {
	bucket string
	key    string
	kind   int
	size   uint64
	body   io.ReadCloser
}

func (ctx s3fs) String() string {
	var kind string
	switch ctx.kind {
	case tUnknown:
		kind = "unknown"
	case tPrefix:
		kind = "prefix"
	case tObject:
		kind = "object"
	}
	return fmt.Sprintf("{key: %#v, kind: %s}", ctx.key, kind)
}

func (ctx s3fs) Root() (fuse.Node, fuse.Error) {
	dprintf("Root: %v", ctx)
	ctx.key = ""
	ctx.kind = tPrefix
	return ctx, nil
}

func (ctx s3fs) Attr() fuse.Attr {
	dprintf("Attr: %v", ctx)
	switch ctx.kind {
	case tPrefix:
		dprintf("Attr: type: directory")
		return fuse.Attr{
			Mode: os.ModeDir | 0755,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	case tObject:
		dprintf("Attr: type: regular file")
		return fuse.Attr{
			Mode: 0644,
			Size: ctx.size,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	default:
		dprintf("Attr: type: unsupported")
		return fuse.Attr{}
	}
}

func (ctx s3fs) Lookup(name string, intr fuse.Intr) (fuse.Node, fuse.Error) {
	dprintf("Lookup: %#v (ctx.key: %#v)", name, ctx.key)
	var key string
	if ctx.key == "" {
		key = name
	} else {
		key = ctx.key + "/" + name
	}
	kind, obj, ok := s3Lookup(key)
	if !ok {
		dprintf("Lookup: returned ENOENT for %#v / %#v", ctx.key, name)
		return nil, fuse.ENOENT
	}
	ctx.key = key
	ctx.kind = kind
	if obj != nil {
		ctx.size = uint64(*obj.ContentLength)
		ctx.body = obj.Body
	} else {
		ctx.size = 0
		ctx.body = nil
	}
	dprintf("Lookup: got %v", ctx)
	return ctx, nil
}

func (ctx s3fs) ReadDir(intr fuse.Intr) ([]fuse.Dirent, fuse.Error) {
	dprintf("ReadDir: %v", ctx)
	if ctx.kind != tPrefix {
		return nil, fuse.EIO
	}

	objs, tPrefixes, err := s3ListDir(ctx.key)
	if err != nil {
		dprintf("ReadDir: %v", err)
	}

	out := make([]fuse.Dirent, 0, len(objs)+len(tPrefixes))
	for _, obj := range objs {
		out = append(out, fuse.Dirent{Name: filepath.Base(obj)})
	}
	for _, pre := range tPrefixes {
		out = append(out, fuse.Dirent{Name: filepath.Base(pre)})
	}

	dprintf("ReadDir: got %d dirents: %v", len(out), out)
	return out, nil
}

func (ctx s3fs) ReadAll(intr fuse.Intr) ([]byte, fuse.Error) {
	dprintf("ReadAll: %v", ctx)
	defer ctx.body.Close()
	buf, err := ioutil.ReadAll(ctx.body)
	if err != nil {
		dprintf("ReadAll: error reading %#v: %v", ctx.key, err)
		return nil, fuse.EIO
	}
	return buf, nil
}

func (ctx s3fs) Create(req *fuse.CreateRequest, res *fuse.CreateResponse, intr fuse.Intr) (fuse.Node, fuse.Handle, fuse.Error) {
	dprintf("Create: %v (ctx.key = %#v)", req, ctx.key)
	if ctx.key == "" {
		ctx.key = req.Name
	} else {
		ctx.key += "/" + req.Name
	}
	if req.Mode&os.ModeDir > 0 {
		ctx.kind = tPrefix
	} else {
		ctx.kind = tObject
	}
	return ctx, ctx, nil
}

func (ctx s3fs) Remove(req *fuse.RemoveRequest, intr fuse.Intr) fuse.Error {
	dprintf("Remove: %v (ctx.key = %#v)", req, ctx.key)
	var key string
	if ctx.key == "" {
		key = req.Name
	} else {
		key = ctx.key + "/" + req.Name
	}
	err := s3RemoveObj(key)
	if err != nil {
		dprintf("Remove: error removing %#v: %v", key, err)
		return fuse.EIO
{'fraction_non_alphanumeric': 0.16900520258552734, 'fraction_numerical': 0.008513321772032161, 'mean_word_length': 3.3511659807956105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '838139', 'n_tokens_mistral': 2845, 'n_tokens_neox': 2653, 'n_words': 685}