<commit_before>/* Copyright (c) 2014-2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type FileStore struct {
	sync.RWMutex
	cache map[ID]fileCache

	dir   string
	stats Stats
}

// Contains static info
type fileCache struct {
	path    string
	modTime time.Time
	size    int64
	reading sync.WaitGroup
}

type FilePaste struct {
	file  *os.File
	cache *fileCache
}

func (c FilePaste) Read(p []byte) (n int, err error) {
	return c.file.Read(p)
}

func (c FilePaste) ReadAt(p []byte, off int64) (n int, err error) {
	return c.file.ReadAt(p, off)
}

func (c FilePaste) Seek(offset int64, whence int) (int64, error) {
	return c.file.Seek(offset, whence)
}

func (c FilePaste) Close() error {
	err := c.file.Close()
	c.cache.reading.Done()
	return err
}

func (c FilePaste) ModTime() time.Time {
	return c.cache.modTime
}

func (c FilePaste) Size() int64 {
	return c.cache.size
}

func NewFileStore(dir string) (*FileStore, error) {
	if err := setupTopDir(dir); err != nil {
		return nil, err
	}
	s := new(FileStore)
	s.dir = dir
	s.cache = make(map[ID]fileCache)
	if err := setupSubdirs(s.dir, s.Recover); err != nil {
		return nil, err
	}
	return s, nil
}

func (s *FileStore) Get(id ID) (Paste, error) {
	s.RLock()
	defer s.RUnlock()
	cached, e := s.cache[id]
	if !e {
		return nil, ErrPasteNotFound
	}
	f, err := os.Open(cached.path)
	if err != nil {
		return nil, err
	}
	cached.reading.Add(1)
	return FilePaste{file: f, cache: &cached}, nil
}

func writeNewFile(filename string, data []byte) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}

func (s *FileStore) Put(content []byte) (ID, error) {
	s.Lock()
	defer s.Unlock()
	size := int64(len(content))
	if err := s.stats.hasSpaceFor(size); err != nil {
		return ID{}, err
	}
	available := func(id ID) bool {
		_, e := s.cache[id]
		return !e
	}
	id, err := randomID(available)
	if err != nil {
		return id, err
	}
	pastePath := pathFromID(id)
	if err = writeNewFile(pastePath, content); err != nil {
		return id, err
	}
	s.stats.makeSpaceFor(size)
	s.cache[id] = fileCache{
		path:    pastePath,
		size:    size,
		modTime: time.Now(),
	}
	return id, nil
}

func (s *FileStore) Delete(id ID) error {
	s.Lock()
	defer s.Unlock()
	cached, e := s.cache[id]
	if !e {
		return ErrPasteNotFound
	}
	delete(s.cache, id)
	cached.reading.Wait()
	if err := os.Remove(cached.path); err != nil {
		return err
	}
	s.stats.freeSpace(cached.size)
	return nil
}

func pathFromID(id ID) string {
	hexID := id.String()
	return path.Join(hexID[:2], hexID[2:])
}

func idFromPath(path string) (ID, error) {
	parts := strings.Split(path, string(filepath.Separator))
	if len(parts) != 2 {
		return ID{}, fmt.Errorf("invalid number of directories at %s", path)
	}<commit_msg>Error if we find a directory that is too short or long<commit_after>/* Copyright (c) 2014-2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type FileStore struct {
	sync.RWMutex
	cache map[ID]fileCache

	dir   string
	stats Stats
}

// Contains static info
type fileCache struct {
	path    string
	modTime time.Time
	size    int64
	reading sync.WaitGroup
}

type FilePaste struct {
	file  *os.File
	cache *fileCache
}

func (c FilePaste) Read(p []byte) (n int, err error) {
	return c.file.Read(p)
}

func (c FilePaste) ReadAt(p []byte, off int64) (n int, err error) {
	return c.file.ReadAt(p, off)
}

func (c FilePaste) Seek(offset int64, whence int) (int64, error) {
	return c.file.Seek(offset, whence)
}

func (c FilePaste) Close() error {
	err := c.file.Close()
	c.cache.reading.Done()
	return err
}

func (c FilePaste) ModTime() time.Time {
	return c.cache.modTime
}

func (c FilePaste) Size() int64 {
	return c.cache.size
}

func NewFileStore(dir string) (*FileStore, error) {
	if err := setupTopDir(dir); err != nil {
		return nil, err
	}
	s := new(FileStore)
	s.dir = dir
	s.cache = make(map[ID]fileCache)
	if err := setupSubdirs(s.dir, s.Recover); err != nil {
		return nil, err
	}
	return s, nil
}

func (s *FileStore) Get(id ID) (Paste, error) {
	s.RLock()
	defer s.RUnlock()
	cached, e := s.cache[id]
	if !e {
		return nil, ErrPasteNotFound
	}
	f, err := os.Open(cached.path)
	if err != nil {
		return nil, err
	}
	cached.reading.Add(1)
	return FilePaste{file: f, cache: &cached}, nil
}

func writeNewFile(filename string, data []byte) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}

func (s *FileStore) Put(content []byte) (ID, error) {
	s.Lock()
	defer s.Unlock()
	size := int64(len(content))
	if err := s.stats.hasSpaceFor(size); err != nil {
		return ID{}, err
	}
	available := func(id ID) bool {
		_, e := s.cache[id]
		return !e
	}
	id, err := randomID(available)
	if err != nil {
		return id, err
	}
	pastePath := pathFromID(id)
	if err = writeNewFile(pastePath, content); err != nil {
		return id, err
	}
	s.stats.makeSpaceFor(size)
	s.cache[id] = fileCache{
		path:    pastePath,
		size:    size,
		modTime: time.Now(),
	}
	return id, nil
}

func (s *FileStore) Delete(id ID) error {
	s.Lock()
	defer s.Unlock()
	cached, e := s.cache[id]
	if !e {
		return ErrPasteNotFound
	}
	delete(s.cache, id)
	cached.reading.Wait()
	if err := os.Remove(cached.path); err != nil {
		return err
	}
	s.stats.freeSpace(cached.size)
	return nil
}

func pathFromID(id ID) string {
	hexID := id.String()
	return path.Join(hexID[:2], hexID[2:])
}

func idFromPath(path string) (ID, error) {
	parts := strings.Split(path, string(filepath.Separator))
	if len(parts) != 2 {
		return ID{}, fmt.Errorf("invalid number of directories at %s", path)
	}
	if len(parts[0]) != 2 {
		return ID{}, fmt.Errorf("invalid directory name length at %s", path)
	}
{'fraction_non_alphanumeric': 0.15505115089514065, 'fraction_numerical': 0.009910485933503837, 'mean_word_length': 3.587243401759531, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4350660', 'n_tokens_mistral': 2588, 'n_tokens_neox': 2496, 'n_words': 689}