<commit_before>package filecache

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"
)

const VERSION = "1.0.2"

// File size constants for use with FileCache.MaxSize.
// For example, cache.MaxSize = 64 * Megabyte
const (
	Kilobyte = 1024
	Megabyte = 1024 * 1024
	Gigabyte = 1024 * 1024 * 1024
)

var (
	DefaultExpireItem int   = 300 // 5 minutes
	DefaultMaxSize    int64 = 16 * Megabyte
	DefaultMaxItems   int   = 32
	DefaultEvery      int   = 60 // 1 minute
)

var (
	InvalidCacheItem = errors.New("invalid cache item")
	ItemIsDirectory  = errors.New("can't cache a directory")
	ItemNotInCache   = errors.New("item not in cache")
	ItemTooLarge     = errors.New("item too large for cache")
	WriteIncomplete  = errors.New("incomplete write of cache item")
)

var SquelchItemNotInCache = true

// Mumber of items to buffer adding to the file cache.
var NewCachePipeSize = 4

type cacheItem struct {
	content    []byte
	lock       sync.Mutex
	Size       int64
	Lastaccess time.Time
	Modified   time.Time
}

func (itm *cacheItem) Lock() {
	itm.lock.Lock()
}

func (itm *cacheItem) Unlock() {
	itm.lock.Unlock()
}

func (itm *cacheItem) WasModified(fi os.FileInfo) bool {
	itm.Lock()
	defer itm.Unlock()
	return itm.Modified.Equal(fi.ModTime())
}

func (itm *cacheItem) GetReader() io.Reader {
	b := bytes.NewReader(itm.Access())
	return b
}

func (itm *cacheItem) Access() []byte {
	itm.Lock()
	defer itm.Unlock()
	itm.Lastaccess = time.Now()
	return itm.content
}

func (itm *cacheItem) Dur() time.Duration {
        itm.Lock()
        defer itm.Unlock()
        return time.Now().Sub(itm.Lastaccess)
}

func cacheFile(path string, maxSize int64) (itm *cacheItem, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return
	} else if fi.Mode().IsDir() {
		return nil, ItemIsDirectory
	} else if fi.Size() > maxSize {
		return nil, ItemTooLarge
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	itm = &cacheItem{
		content:    content,
		Size:       fi.Size(),
		Modified:   fi.ModTime(),
		Lastaccess: time.Now(),<commit_msg>forcibly prevent callers from locking item<commit_after>package filecache

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"
)

const VERSION = "1.0.2"

// File size constants for use with FileCache.MaxSize.
// For example, cache.MaxSize = 64 * Megabyte
const (
	Kilobyte = 1024
	Megabyte = 1024 * 1024
	Gigabyte = 1024 * 1024 * 1024
)

var (
	DefaultExpireItem int   = 300 // 5 minutes
	DefaultMaxSize    int64 = 16 * Megabyte
	DefaultMaxItems   int   = 32
	DefaultEvery      int   = 60 // 1 minute
)

var (
	InvalidCacheItem = errors.New("invalid cache item")
	ItemIsDirectory  = errors.New("can't cache a directory")
	ItemNotInCache   = errors.New("item not in cache")
	ItemTooLarge     = errors.New("item too large for cache")
	WriteIncomplete  = errors.New("incomplete write of cache item")
)

var SquelchItemNotInCache = true

// Mumber of items to buffer adding to the file cache.
var NewCachePipeSize = 4

type cacheItem struct {
	content    []byte
	lock       sync.Mutex
	Size       int64
	Lastaccess time.Time
	Modified   time.Time
}

func (itm *cacheItem) WasModified(fi os.FileInfo) bool {
	itm.lock.Lock()
	defer itm.lock.Unlock()
	return itm.Modified.Equal(fi.ModTime())
}

func (itm *cacheItem) GetReader() io.Reader {
	b := bytes.NewReader(itm.Access())
	return b
}

func (itm *cacheItem) Access() []byte {
	itm.lock.Lock()
	defer itm.lock.Unlock()
	itm.Lastaccess = time.Now()
	return itm.content
}

func (itm *cacheItem) Dur() time.Duration {
        itm.lock.Lock()
        defer itm.lock.Unlock()
        return time.Now().Sub(itm.Lastaccess)
}

func cacheFile(path string, maxSize int64) (itm *cacheItem, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return
	} else if fi.Mode().IsDir() {
		return nil, ItemIsDirectory
	} else if fi.Size() > maxSize {
		return nil, ItemTooLarge
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	itm = &cacheItem{
		content:    content,
		Size:       fi.Size(),
		Modified:   fi.ModTime(),
		Lastaccess: time.Now(),
{'fraction_non_alphanumeric': 0.12822332623610125, 'fraction_numerical': 0.02223799384906553, 'mean_word_length': 3.3054989816700613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6760146', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1566, 'n_words': 420}