<commit_before>package image

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/werf/lockgate"
	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/slug"
	"github.com/werf/werf/pkg/util"
	"github.com/werf/werf/pkg/werf"
)

const (
	ManifestCacheVersion = "4"
)

type ManifestCache struct {
	CacheDir string
}

type ManifestCacheRecord struct {
	AccessTimestamp int64
	Info            *Info
}

func NewManifestCache(cacheDir string) *ManifestCache {
	return &ManifestCache{CacheDir: cacheDir}
}

func (cache *ManifestCache) GetImageInfo(ctx context.Context, storageName, imageName string) (*Info, error) {
	logProcess := logboek.Context(ctx).Debug().LogProcess("-- ManifestCache.GetImageInfo %s %s", storageName, imageName)
	logProcess.Start()
	defer logProcess.End()

	if lock, err := cache.lock(ctx, storageName, imageName); err != nil {
		return nil, err
	} else {
		defer cache.unlock(lock)
	}

	now := time.Now()

	if record, err := cache.readRecord(storageName, imageName); err != nil {
		return nil, err
	} else if record != nil {
		record.AccessTimestamp = now.Unix()
		if err := cache.writeRecord(storageName, record); err != nil {
			return nil, err
		}
		return record.Info, nil
	} else {
		return nil, nil
	}
}

func (cache *ManifestCache) StoreImageInfo(ctx context.Context, storageName string, imgInfo *Info) error {
	logProcess := logboek.Context(ctx).Debug().LogProcess("-- ManifestCache.StoreImageInfo %s %s", storageName, imgInfo.Name)
	logProcess.Start()
	defer logProcess.End()

	if lock, err := cache.lock(ctx, storageName, imgInfo.Name); err != nil {
		return err
	} else {
		defer cache.unlock(lock)
	}

	record := &ManifestCacheRecord{
		AccessTimestamp: time.Now().Unix(),
		Info:            imgInfo,
	}
	return cache.writeRecord(storageName, record)
}

func (cache *ManifestCache) readRecord(storageName, imageName string) (*ManifestCacheRecord, error) {
	filePath := cache.constructFilePathForImage(storageName, imageName)

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, fmt.Errorf("error accessing %s: %s", filePath, err)
	}

	if dataBytes, err := ioutil.ReadFile(filePath); err != nil {
		return nil, fmt.Errorf("error reading %s: %s", filePath, err)
	} else {
		record := &ManifestCacheRecord{}
		if err := json.Unmarshal(dataBytes, record); err != nil {
			return nil, fmt.Errorf("error unmarshalling json from %s: %s", filePath, err)
		}
		return record, nil
	}
}

func (cache *ManifestCache) writeRecord(storageName string, record *ManifestCacheRecord) error {
	filePath := cache.constructFilePathForImage(storageName, record.Info.Name)

	dirPath := filepath.Dir(filePath)
	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
		return fmt.Errorf("error creating dir %s: %s", dirPath, err)
	}

	if dataBytes, err := json.Marshal(record); err != nil {
		return fmt.Errorf("error marshalling json: %s", err)
	} else {
		if err := ioutil.WriteFile(filePath, append(dataBytes, []byte("\n")...), 0644); err != nil {
			return fmt.Errorf("error writing %s: %s", filePath, err)
		}
		return nil
	}
}

func (cache *ManifestCache) constructFilePathForImage(storageName, imageName string) string {
	return filepath.Join(cache.CacheDir, slug.Slug(storageName), util.Sha256Hash(imageName))
}
<commit_msg>Automatically invalidate corrupted manifests cache json records<commit_after>package image

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/werf/lockgate"
	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/slug"
	"github.com/werf/werf/pkg/util"
	"github.com/werf/werf/pkg/werf"
)

const (
	ManifestCacheVersion = "4"
)

type ManifestCache struct {
	CacheDir string
}

type ManifestCacheRecord struct {
	AccessTimestamp int64
	Info            *Info
}

func NewManifestCache(cacheDir string) *ManifestCache {
	return &ManifestCache{CacheDir: cacheDir}
}

func (cache *ManifestCache) GetImageInfo(ctx context.Context, storageName, imageName string) (*Info, error) {
	logProcess := logboek.Context(ctx).Debug().LogProcess("-- ManifestCache.GetImageInfo %s %s", storageName, imageName)
	logProcess.Start()
	defer logProcess.End()

	if lock, err := cache.lock(ctx, storageName, imageName); err != nil {
		return nil, err
	} else {
		defer cache.unlock(lock)
	}

	now := time.Now()

	if record, err := cache.readRecord(ctx, storageName, imageName); err != nil {
		return nil, err
	} else if record != nil {
		record.AccessTimestamp = now.Unix()
		if err := cache.writeRecord(storageName, record); err != nil {
			return nil, err
		}
		return record.Info, nil
	} else {
		return nil, nil
	}
}

func (cache *ManifestCache) StoreImageInfo(ctx context.Context, storageName string, imgInfo *Info) error {
	logProcess := logboek.Context(ctx).Debug().LogProcess("-- ManifestCache.StoreImageInfo %s %s", storageName, imgInfo.Name)
	logProcess.Start()
	defer logProcess.End()

	if lock, err := cache.lock(ctx, storageName, imgInfo.Name); err != nil {
		return err
	} else {
		defer cache.unlock(lock)
	}

	record := &ManifestCacheRecord{
		AccessTimestamp: time.Now().Unix(),
		Info:            imgInfo,
	}
	return cache.writeRecord(storageName, record)
}

func (cache *ManifestCache) readRecord(ctx context.Context, storageName, imageName string) (*ManifestCacheRecord, error) {
	filePath := cache.constructFilePathForImage(storageName, imageName)

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, fmt.Errorf("error accessing %s: %s", filePath, err)
	}

	if dataBytes, err := ioutil.ReadFile(filePath); err != nil {
		return nil, fmt.Errorf("error reading %s: %s", filePath, err)
	} else {
		record := &ManifestCacheRecord{}
		if err := json.Unmarshal(dataBytes, record); err != nil {
			logboek.Context(ctx).Error().LogF("WARNING: invalid manifests cache json record in file %s: %s: resetting record\n", filePath, err)
			return nil, nil
		}
		return record, nil
	}
}

func (cache *ManifestCache) writeRecord(storageName string, record *ManifestCacheRecord) error {
	filePath := cache.constructFilePathForImage(storageName, record.Info.Name)

	dirPath := filepath.Dir(filePath)
	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
		return fmt.Errorf("error creating dir %s: %s", dirPath, err)
	}

	if dataBytes, err := json.Marshal(record); err != nil {
		return fmt.Errorf("error marshalling json: %s", err)
	} else {
		if err := ioutil.WriteFile(filePath, append(dataBytes, []byte("\n")...), 0644); err != nil {
			return fmt.Errorf("error writing %s: %s", filePath, err)
		}
		return nil
	}
}

func (cache *ManifestCache) constructFilePathForImage(storageName, imageName string) string {
	return filepath.Join(cache.CacheDir, slug.Slug(storageName), util.Sha256Hash(imageName))
}

{'fraction_non_alphanumeric': 0.13886858646396727, 'fraction_numerical': 0.0029235491887151, 'mean_word_length': 4.553571428571429, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2459720', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2322, 'n_words': 578}