<commit_before>package caching

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws/session"
	aw "github.com/deanishe/awgo"

	"github.com/cheekybits/genny/generic"
)

//go:generate genny -in=$GOFILE -out=gen-$GOFILE gen "Entity=ec2.Instance,s3.Bucket,ec2.SecurityGroup,elasticbeanstalk.EnvironmentDescription"
type Entity = generic.Type

type EntityArrayFetcher = func(*session.Session) ([]Entity, error)

func LoadEntityArrayFromCache(wf *aw.Workflow, session *session.Session, cacheName string, fetcher EntityArrayFetcher, forceFetch bool, fullQuery string) []Entity {
	// TODO optimization: not all services have sa region associated with them, so cache can be reused across regions (e.g., s3 buckets are global)
	cacheName += "_" + *session.Config.Region

	results := []Entity{}
	lastFetchErrPath := wf.CacheDir() + "/last-fetch-err.txt"
	if forceFetch {
		log.Printf("fetching from aws ...")
		results, err := fetcher(session)

		if err != nil {
			log.Printf("fetch error occurred. writing to %s ...", lastFetchErrPath)
			ioutil.WriteFile(lastFetchErrPath, []byte(err.Error()), 0600)
			panic(err)
		} else {
			os.Remove(lastFetchErrPath)
		}
		log.Printf("fetched %d results from aws", len(results))

		log.Printf("storing %d results with cache key `%s` to %s ...", len(results), cacheName, wf.CacheDir())<commit_msg>fixed lint<commit_after>package caching

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws/session"
	aw "github.com/deanishe/awgo"

	"github.com/cheekybits/genny/generic"
)

//go:generate genny -in=$GOFILE -out=gen-$GOFILE gen "Entity=ec2.Instance,s3.Bucket,ec2.SecurityGroup,elasticbeanstalk.EnvironmentDescription"
type Entity = generic.Type

type EntityArrayFetcher = func(*session.Session) ([]Entity, error)

func LoadEntityArrayFromCache(wf *aw.Workflow, session *session.Session, cacheName string, fetcher EntityArrayFetcher, forceFetch bool, fullQuery string) []Entity {
	// TODO optimization: not all services have sa region associated with them, so cache can be reused across regions (e.g., s3 buckets are global)
	cacheName += "_" + *session.Config.Region

	results := []Entity{}
	lastFetchErrPath := wf.CacheDir() + "/last-fetch-err.txt"
	if forceFetch {
		log.Printf("fetching from aws ...")
		results, err := fetcher(session)

		if err != nil {
			log.Printf("fetch error occurred. writing to %s ...", lastFetchErrPath)
			_ = ioutil.WriteFile(lastFetchErrPath, []byte(err.Error()), 0600)
			panic(err)
		} else {
			os.Remove(lastFetchErrPath)
		}
		log.Printf("fetched %d results from aws", len(results))

		log.Printf("storing %d results with cache key `%s` to %s ...", len(results), cacheName, wf.CacheDir())
{'fraction_non_alphanumeric': 0.14264432029795157, 'fraction_numerical': 0.0059590316573556795, 'mean_word_length': 4.690677966101695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2925453', 'n_tokens_mistral': 953, 'n_tokens_neox': 908, 'n_words': 220}