<commit_before>package trustmanager

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

const visible os.FileMode = 0755
const private os.FileMode = 0700

// FileStore is the interface for all FileStores
type FileStore interface {
	Add(fileName string, data []byte) error
	Remove(fileName string) error
	RemoveGUN(gun string) error
	GetData(fileName string) ([]byte, error)
	GetPath(fileName string) string
	List() []string
	ListGUN(gun string) []string
}

type fileStore struct {
	baseDir string
	fileExt string
	perms   os.FileMode
}

func NewFileStore(baseDir string, fileExt string) (FileStore, error) {
	if err := CreateDirectory(baseDir); err != nil {
		return nil, err
	}

	return &fileStore{
		baseDir: baseDir,
		fileExt: fileExt,
		perms:   visible,
	}, nil
}

func NewPrivateFileStore(baseDir string, fileExt string) (FileStore, error) {
	if err := CreatePrivateDirectory(baseDir); err != nil {
		return nil, err
	}

	return &fileStore{
		baseDir: baseDir,
		fileExt: fileExt,
		perms:   private,
	}, nil
}

func (f *fileStore) Add(name string, data []byte) error {
	filePath := f.genFilePath(name)
	createDirectory(filepath.Dir(filePath), f.perms)
	if err := ioutil.WriteFile(filePath, data, f.perms); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) Remove(name string) error {
	filePath := f.genFilePath(name)
	if err := os.Remove(filePath); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) RemoveGUN(gun string) error {
	dirPath := filepath.Join(f.baseDir, gun)

	// Check to see if file exists
	fi, err := os.Stat(dirPath)
	if err != nil {
		return err
	}

	// Check to see if it is a directory
	if !fi.IsDir() {
		return fmt.Errorf("GUN not found: %s", gun)
	}

	if err := os.RemoveAll(dirPath); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) GetData(name string) ([]byte, error) {
	filePath := f.genFilePath(name)
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, err
	}

	return data, nil
}

func (f *fileStore) GetPath(name string) string {
	return f.genFilePath(name)
}

func (f *fileStore) List() []string {
	files := make([]string, 0, 0)
	filepath.Walk(f.baseDir, func(fp string, fi os.FileInfo, err error) error {
		// If there are errors, ignore this particular file
		if err != nil {
			return nil
		}
		// Ignore if it is a directory
		if fi.IsDir() {
			return nil
		}
		// Only allow matches that end with our certificate extension (e.g. *.crt)
		matched, _ := filepath.Match("*"+f.fileExt, fi.Name())

		if matched {
			files = append(files, fp)
		}
		return nil
	})
	return files
}

func (f *fileStore) ListGUN(gun string) []string {
	files := make([]string, 0, 0)
	// This prevents someone passing /path/to/dir and 'dir' not being included
	// If two '//' exist, Walk deals it with correctly
	gunPath := filepath.Join(f.baseDir, gun) + "/"
	filepath.Walk(gunPath, func(fp string, fi os.FileInfo, err error) error {
		// If there are errors, ignore this particular file
		if err != nil {
			return nil
		}
		// Ignore if it is a directory
		if fi.IsDir() {
			return nil<commit_msg>Making ListGUN and List DRY<commit_after>package trustmanager

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

const visible os.FileMode = 0755
const private os.FileMode = 0700

// FileStore is the interface for all FileStores
type FileStore interface {
	Add(fileName string, data []byte) error
	Remove(fileName string) error
	RemoveGUN(gun string) error
	GetData(fileName string) ([]byte, error)
	GetPath(fileName string) string
	List() []string
	ListGUN(gun string) []string
}

type fileStore struct {
	baseDir string
	fileExt string
	perms   os.FileMode
}

func NewFileStore(baseDir string, fileExt string) (FileStore, error) {
	if err := CreateDirectory(baseDir); err != nil {
		return nil, err
	}

	return &fileStore{
		baseDir: baseDir,
		fileExt: fileExt,
		perms:   visible,
	}, nil
}

func NewPrivateFileStore(baseDir string, fileExt string) (FileStore, error) {
	if err := CreatePrivateDirectory(baseDir); err != nil {
		return nil, err
	}

	return &fileStore{
		baseDir: baseDir,
		fileExt: fileExt,
		perms:   private,
	}, nil
}

func (f *fileStore) Add(name string, data []byte) error {
	filePath := f.genFilePath(name)
	createDirectory(filepath.Dir(filePath), f.perms)
	if err := ioutil.WriteFile(filePath, data, f.perms); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) Remove(name string) error {
	filePath := f.genFilePath(name)
	if err := os.Remove(filePath); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) RemoveGUN(gun string) error {
	dirPath := filepath.Join(f.baseDir, gun)

	// Check to see if file exists
	fi, err := os.Stat(dirPath)
	if err != nil {
		return err
	}

	// Check to see if it is a directory
	if !fi.IsDir() {
		return fmt.Errorf("GUN not found: %s", gun)
	}

	if err := os.RemoveAll(dirPath); err != nil {
		return err
	}

	return nil
}

func (f *fileStore) GetData(name string) ([]byte, error) {
	filePath := f.genFilePath(name)
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, err
	}

	return data, nil
}

func (f *fileStore) GetPath(name string) string {
	return f.genFilePath(name)
}

func (f *fileStore) List() []string {
	return f.listGUN(f.baseDir)
}

func (f *fileStore) ListGUN(gun string) []string {
	gunPath := filepath.Join(f.baseDir, gun)
	return f.listGUN(gunPath)
}

func (f *fileStore) listGUN(gunPath string) []string {
	files := make([]string, 0, 0)
	filepath.Walk(gunPath, func(fp string, fi os.FileInfo, err error) error {
		// If there are errors, ignore this particular file
		if err != nil {
			return nil
		}
		// Ignore if it is a directory
		if fi.IsDir() {
			return nil
{'fraction_non_alphanumeric': 0.12603269467393216, 'fraction_numerical': 0.0038671119704693268, 'mean_word_length': 3.7895622895622894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6169431', 'n_tokens_mistral': 2126, 'n_tokens_neox': 2029, 'n_words': 631}