<commit_before>package block

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/kopia/kopia/storage"
	"github.com/rs/zerolog/log"
)

type listCache struct {
	st                storage.Storage
	cacheFile         string
	listCacheDuration time.Duration
	hmacSecret        []byte
}

func (c *listCache) listIndexBlocks(ctx context.Context) ([]IndexInfo, error) {
	if c.cacheFile != "" {
		ci, err := c.readBlocksFromCache(ctx)
		if err == nil {
			expirationTime := ci.Timestamp.Add(c.listCacheDuration)
			if time.Now().Before(expirationTime) {
				log.Debug().Msg("retrieved list of index blocks from cache")
				return ci.Blocks, nil
			}
		} else if err != storage.ErrBlockNotFound {
			log.Warn().Err(err).Msgf("unable to open cache file")
		}
	}

	blocks, err := listIndexBlocksFromStorage(ctx, c.st)
	if err == nil {
		c.saveListToCache(ctx, &cachedList{
			Blocks:    blocks,
			Timestamp: time.Now(),
		})
	}
	log.Debug().Msgf("found %v index blocks from source", len(blocks))

	return blocks, err
}

func (c *listCache) saveListToCache(ctx context.Context, ci *cachedList) {
	if c.cacheFile == "" {
		return
	}
	log.Debug().Int("count", len(ci.Blocks)).Msg("saving index blocks to cache")
	if data, err := json.Marshal(ci); err == nil {
		if err := ioutil.WriteFile(c.cacheFile, appendHMAC(data, c.hmacSecret), 0600); err != nil {
			log.Warn().Msgf("unable to write list cache: %v", err)
		}
	}
}

func (c *listCache) deleteListCache(ctx context.Context) {
	if c.cacheFile != "" {
		os.Remove(c.cacheFile) //nolint:errcheck
	}
}

func (c *listCache) readBlocksFromCache(ctx context.Context) (*cachedList, error) {
	if !shouldUseListCache(ctx) {
		return nil, storage.ErrBlockNotFound
	}

	ci := &cachedList{}

	data, err := ioutil.ReadFile(c.cacheFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, storage.ErrBlockNotFound
		}

		return nil, err
	}

	data, err = verifyAndStripHMAC(data, c.hmacSecret)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(data, &ci); err != nil {
		return nil, fmt.Errorf("can't unmarshal cached list results: %v", err)
	}

	return ci, nil

}

func newListCache(ctx context.Context, st storage.Storage, caching CachingOptions) (*listCache, error) {
	var listCacheFile string

	if caching.CacheDirectory != "" {
		listCacheFile = filepath.Join(caching.CacheDirectory, "list")<commit_msg>write list cache file atomically<commit_after>package block

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/kopia/kopia/storage"
	"github.com/rs/zerolog/log"
)

type listCache struct {
	st                storage.Storage
	cacheFile         string
	listCacheDuration time.Duration
	hmacSecret        []byte
}

func (c *listCache) listIndexBlocks(ctx context.Context) ([]IndexInfo, error) {
	if c.cacheFile != "" {
		ci, err := c.readBlocksFromCache(ctx)
		if err == nil {
			expirationTime := ci.Timestamp.Add(c.listCacheDuration)
			if time.Now().Before(expirationTime) {
				log.Debug().Msg("retrieved list of index blocks from cache")
				return ci.Blocks, nil
			}
		} else if err != storage.ErrBlockNotFound {
			log.Warn().Err(err).Msgf("unable to open cache file")
		}
	}

	blocks, err := listIndexBlocksFromStorage(ctx, c.st)
	if err == nil {
		c.saveListToCache(ctx, &cachedList{
			Blocks:    blocks,
			Timestamp: time.Now(),
		})
	}
	log.Debug().Msgf("found %v index blocks from source", len(blocks))

	return blocks, err
}

func (c *listCache) saveListToCache(ctx context.Context, ci *cachedList) {
	if c.cacheFile == "" {
		return
	}
	log.Debug().Int("count", len(ci.Blocks)).Msg("saving index blocks to cache")
	if data, err := json.Marshal(ci); err == nil {
		mySuffix := fmt.Sprintf(".tmp-%v-%v", os.Getpid(), time.Now().UnixNano())
		if err := ioutil.WriteFile(c.cacheFile+mySuffix, appendHMAC(data, c.hmacSecret), 0600); err != nil {
			log.Warn().Msgf("unable to write list cache: %v", err)
		}
		os.Rename(c.cacheFile+mySuffix, c.cacheFile) //nolint:errcheck
		os.Remove(c.cacheFile + mySuffix)            //nolint:errcheck
	}
}

func (c *listCache) deleteListCache(ctx context.Context) {
	if c.cacheFile != "" {
		os.Remove(c.cacheFile) //nolint:errcheck
	}
}

func (c *listCache) readBlocksFromCache(ctx context.Context) (*cachedList, error) {
	if !shouldUseListCache(ctx) {
		return nil, storage.ErrBlockNotFound
	}

	ci := &cachedList{}

	data, err := ioutil.ReadFile(c.cacheFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, storage.ErrBlockNotFound
		}

		return nil, err
	}

	data, err = verifyAndStripHMAC(data, c.hmacSecret)
	if err != nil {
		return nil, fmt.Errorf("invalid file %v: %v", c.cacheFile, err)
	}

	if err := json.Unmarshal(data, &ci); err != nil {
		return nil, fmt.Errorf("can't unmarshal cached list results: %v", err)
	}

	return ci, nil

}

func newListCache(ctx context.Context, st storage.Storage, caching CachingOptions) (*listCache, error) {
	var listCacheFile string

	if caching.CacheDirectory != "" {
		listCacheFile = filepath.Join(caching.CacheDirectory, "list")
{'fraction_non_alphanumeric': 0.14636542239685657, 'fraction_numerical': 0.0015717092337917485, 'mean_word_length': 3.8717703349282298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2837912', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1802, 'n_words': 424}