<commit_before>package pigae

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

// This source file has a lot of duplicated code : "if cached then return else datastore and cache".
// TODO: find a smarter design for this "proxy" type which applies basically the same behavior to
// all read methods, and the same behavior to all write methods.

// MemcacheDatastoreAccessor accessor uses a MemCache for standard CRUD.
//
// Some methods are not redefined : randomIdiom, nextIdiomID, nextImplID, processUploadFile, processUploadFiles
type MemcacheDatastoreAccessor struct {
	dataAccessor
}

func (a *MemcacheDatastoreAccessor) cacheValue(c context.Context, cacheKey string, data interface{}, expiration time.Duration) error {
	var buffer bytes.Buffer
	enc := gob.NewEncoder(&buffer)

	err := enc.Encode(&data)
	if err != nil {
		log.Debugf(c, "Failed encoding for cache[%v] : %v", cacheKey, err)
		return err
	}
	cacheItem := &memcache.Item{
		Key:        cacheKey,
		Value:      buffer.Bytes(),
		Expiration: expiration,
	}
	// Set the item, unconditionally
	err = memcache.Set(c, cacheItem)
	if err != nil {
		log.Debugf(c, "Failed setting cache[%v] : %v", cacheKey, err)
	} else {
		// log.Debugf(c, "Successfully set cache[%v]", cacheKey)
	}
	return err
}

func (a *MemcacheDatastoreAccessor) cacheValues(c context.Context, cacheKeys []string, data []interface{}, expiration time.Duration) error {
	if len(cacheKeys) != len(data) {
		panic(fmt.Errorf("Wrong params length", len(cacheKeys), len(data)))
	}
	N := len(cacheKeys)

	var buffer bytes.Buffer
	enc := gob.NewEncoder(&buffer)

	items := make([]*memcache.Item, N)
	for i, cacheKey := range cacheKeys {
		cacheData := data[i]
		err := enc.Encode(&cacheData)
		if err != nil {
			log.Debugf(c, "Failed encoding for cache[%v] : %v", cacheKey, err)
			return err
		}
		cacheItem := &memcache.Item{
			Key:        cacheKey,
			Value:      buffer.Bytes(),
			Expiration: expiration,
		}
		items[i] = cacheItem
	}

	// Set the items, unconditionally, in 1 batch call
	err := memcache.SetMulti(c, items)
	if err != nil {
		log.Debugf(c, "Failed setting cache items: %v", cacheKeys, err)
	}
	return err
}

// Just a shortcut for caching the datastoreKey + value
func (a *MemcacheDatastoreAccessor) cacheKeyValue(c context.Context, cacheKey string, datastoreKey *datastore.Key, entity interface{}, expiration time.Duration) error {
	kae := &KeyAndEntity{datastoreKey, entity}
	return a.cacheValue(c, cacheKey, kae, expiration)
}

// Just a shortcut for caching the datastoreKeys + values
func (a *MemcacheDatastoreAccessor) cacheKeysValues(c context.Context, cacheKeys []string, datastoreKeys []*datastore.Key, entities []interface{}, expiration time.Duration) error {
	if len(cacheKeys) != len(datastoreKeys) || len(cacheKeys) != len(entities) {
		panic(fmt.Errorf("Wrong params length", len(cacheKeys), len(datastoreKeys), len(entities)))
	}
	N := len(cacheKeys)

	items := make([]interface{}, N)
	for i, datastoreKey := range datastoreKeys {
		kae := &KeyAndEntity{datastoreKey, entities[i]}
		items[i] = kae
	}
	return a.cacheValues(c, cacheKeys, items, expiration)
}

// Just a shortcut for caching the pair
func (a *MemcacheDatastoreAccessor) cachePair(c context.Context, cacheKey string, first interface{}, second interface{}, expiration time.Duration) error {
	pair := &pair{first, second}
	return a.cacheValue(c, cacheKey, pair, expiration)
}

func (a *MemcacheDatastoreAccessor) readCache(c context.Context, cacheKey string) (interface{}, error) {
	// Get the item from the memcache
	var cacheItem *memcache.Item
	var err error
	if cacheItem, err = memcache.Get(c, cacheKey); err == memcache.ErrCacheMiss {
		// Item not in the cache
		return nil, nil
	} else if err != nil {
		return nil, err
	} else {
		buffer := bytes.NewBuffer(cacheItem.Value) // todo avoid bytes copy ??
		dec := gob.NewDecoder(buffer)
		var data interface{}
		err = dec.Decode(&data)
		return data, err
	}
}

func init() {
	gob.Register(&KeyAndEntity{})
	gob.Register(&pair{})
	gob.Register(&Idiom{})
	gob.Register([]*datastore.Key{})
	gob.Register([]*Idiom{})
	gob.Register([]string{})
	gob.Register(map[string]bool{})
	gob.Register(Toggles{})
	gob.Register(&ApplicationConfig{})
	gob.Register([]*MessageForUser{})
}

// KeyAndEntity is a specific pair wrapper.
type KeyAndEntity struct {
	Key    *datastore.Key
	Entity interface{}
}

type pair struct {
	First  interface{}
	Second interface{}
}

func (a *MemcacheDatastoreAccessor) recacheIdiom(c context.Context, key *datastore.Key, idiom *Idiom) error {
	cacheKey := fmt.Sprintf("getIdiom(%v)", idiom.Id)
	err := a.cacheKeyValue(c, cacheKey, key, idiom, 24*time.Hour)
	if err != nil {
		log.Errorf(c, err.Error())
		return err
	}

	// Batch memcache set call
	N := len(idiom.Implementations)
	cacheKeys := make([]string, N)
	datastoreKeys := make([]*datastore.Key, N)
	cacheValues := make([]interface{}, N)
	for i, impl := range idiom.Implementations {
		cacheKeys[i] = fmt.Sprintf("getIdiomByImplID(%v)", impl.Id)
		datastoreKeys[i] = key
		cacheValues[i] = idiom
	}
	err = a.cacheKeysValues(c, cacheKeys, datastoreKeys, cacheValues, 24*time.Hour)
	if err != nil {
		log.Errorf(c, err.Error())
		return err
	}
	// Unfortunatly, some previous "getIdiomByImplID(xyz)" might be left uninvalidated.
	// (theoretically)
	return err
}

func (a *MemcacheDatastoreAccessor) uncacheIdiom(c context.Context, idiom *Idiom) error {
	cacheKeys := make([]string, 1+len(idiom.Implementations))
	cacheKeys[0] = fmt.Sprintf("getIdiom(%v)", idiom.Id)
	for i, impl := range idiom.Implementations {
		cacheKeys[1+i] = fmt.Sprintf("getIdiomByImplID(%v)", impl.Id)
	}

	err := memcache.DeleteMulti(c, cacheKeys)
	if err != nil {
		log.Errorf(c, err.Error())
	}
	return err
}

func (a *MemcacheDatastoreAccessor) getIdiom(c context.Context, idiomID int) (*datastore.Key, *Idiom, error) {
	cacheKey := fmt.Sprintf("getIdiom(%v)", idiomID)
	data, cacheerr := a.readCache(c, cacheKey)
	if cacheerr != nil {
		log.Errorf(c, cacheerr.Error())
		// Ouch. Well, skip the cache if it's broken
		return a.dataAccessor.getIdiom(c, idiomID)
	}<commit_msg>Memcache cacheSameValues: save costly encoding calls.<commit_after>package pigae

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

// This source file has a lot of duplicated code : "if cached then return else datastore and cache".
// TODO: find a smarter design for this "proxy" type which applies basically the same behavior to
// all read methods, and the same behavior to all write methods.

// MemcacheDatastoreAccessor accessor uses a MemCache for standard CRUD.
//
// Some methods are not redefined : randomIdiom, nextIdiomID, nextImplID, processUploadFile, processUploadFiles
type MemcacheDatastoreAccessor struct {
	dataAccessor
}

func (a *MemcacheDatastoreAccessor) cacheValue(c context.Context, cacheKey string, data interface{}, expiration time.Duration) error {
	var buffer bytes.Buffer
	enc := gob.NewEncoder(&buffer)

	err := enc.Encode(&data)
	if err != nil {
		log.Debugf(c, "Failed encoding for cache[%v] : %v", cacheKey, err)
		return err
	}
	cacheItem := &memcache.Item{
		Key:        cacheKey,
		Value:      buffer.Bytes(),
		Expiration: expiration,
	}
	// Set the item, unconditionally
	err = memcache.Set(c, cacheItem)
	if err != nil {
		log.Debugf(c, "Failed setting cache[%v] : %v", cacheKey, err)
	} else {
		// log.Debugf(c, "Successfully set cache[%v]", cacheKey)
	}
	return err
}

func (a *MemcacheDatastoreAccessor) cacheValues(c context.Context, cacheKeys []string, data []interface{}, expiration time.Duration) error {
	if len(cacheKeys) != len(data) {
		panic(fmt.Errorf("Wrong params length", len(cacheKeys), len(data)))
	}
	N := len(cacheKeys)

	var buffer bytes.Buffer
	enc := gob.NewEncoder(&buffer)

	items := make([]*memcache.Item, N)
	for i, cacheKey := range cacheKeys {
		cacheData := data[i]
		err := enc.Encode(&cacheData)
		if err != nil {
			log.Debugf(c, "Failed encoding for cache[%v] : %v", cacheKey, err)
			return err
		}
		cacheItem := &memcache.Item{
			Key:        cacheKey,
			Value:      buffer.Bytes(),
			Expiration: expiration,
		}
		items[i] = cacheItem
	}

	// Set the items, unconditionally, in 1 batch call
	err := memcache.SetMulti(c, items)
	if err != nil {
		log.Debugf(c, "Failed setting cache items: %v", cacheKeys, err)
	}
	return err
}

func (a *MemcacheDatastoreAccessor) cacheSameValues(c context.Context, cacheKeys []string, data interface{}, expiration time.Duration) error {
	N := len(cacheKeys)

	var buffer bytes.Buffer
	enc := gob.NewEncoder(&buffer)
	err := enc.Encode(&data)

	if err != nil {
		log.Debugf(c, "Failed encoding for cache keys [%v] : %v", cacheKeys, err)
		return err
	}

	items := make([]*memcache.Item, N)
	for i, cacheKey := range cacheKeys {
		cacheItem := &memcache.Item{
			Key:        cacheKey,
			Value:      buffer.Bytes(),
			Expiration: expiration,
		}
		items[i] = cacheItem
	}

	// Set the items, unconditionally, in 1 batch call
	err = memcache.SetMulti(c, items)
	if err != nil {
		log.Debugf(c, "Failed setting cache items: %v", cacheKeys, err)
	}
	return err
}

// A shortcut for caching the datastoreKey + value
func (a *MemcacheDatastoreAccessor) cacheKeyValue(c context.Context, cacheKey string, datastoreKey *datastore.Key, entity interface{}, expiration time.Duration) error {
	kae := &KeyAndEntity{datastoreKey, entity}
	return a.cacheValue(c, cacheKey, kae, expiration)
}

// A shortcut for caching the datastoreKeys + values
func (a *MemcacheDatastoreAccessor) cacheKeysValues(c context.Context, cacheKeys []string, datastoreKeys []*datastore.Key, entities []interface{}, expiration time.Duration) error {
	if len(cacheKeys) != len(datastoreKeys) || len(cacheKeys) != len(entities) {
		panic(fmt.Errorf("Wrong params length", len(cacheKeys), len(datastoreKeys), len(entities)))
	}
	N := len(cacheKeys)

	items := make([]interface{}, N)
	for i, datastoreKey := range datastoreKeys {
		kae := &KeyAndEntity{datastoreKey, entities[i]}
		items[i] = kae
	}
	return a.cacheValues(c, cacheKeys, items, expiration)
}

// A shortcut for caching the same datastore key and same value (to encode once) for each cacheKey
func (a *MemcacheDatastoreAccessor) cacheKeysSameValue(c context.Context, cacheKeys []string, datastoreKey *datastore.Key, entity interface{}, expiration time.Duration) error {
	kae := &KeyAndEntity{datastoreKey, entity}
	return a.cacheSameValues(c, cacheKeys, kae, expiration)
}

// Just a shortcut for caching the pair
func (a *MemcacheDatastoreAccessor) cachePair(c context.Context, cacheKey string, first interface{}, second interface{}, expiration time.Duration) error {
	pair := &pair{first, second}
	return a.cacheValue(c, cacheKey, pair, expiration)
}

func (a *MemcacheDatastoreAccessor) readCache(c context.Context, cacheKey string) (interface{}, error) {
	// Get the item from the memcache
	var cacheItem *memcache.Item
	var err error
	if cacheItem, err = memcache.Get(c, cacheKey); err == memcache.ErrCacheMiss {
		// Item not in the cache
		return nil, nil
	} else if err != nil {
		return nil, err
	} else {
		buffer := bytes.NewBuffer(cacheItem.Value) // todo avoid bytes copy ??
		dec := gob.NewDecoder(buffer)
		var data interface{}
		err = dec.Decode(&data)
		return data, err
	}
}

func init() {
	gob.Register(&KeyAndEntity{})
	gob.Register(&pair{})
	gob.Register(&Idiom{})
	gob.Register([]*datastore.Key{})
	gob.Register([]*Idiom{})
	gob.Register([]string{})
	gob.Register(map[string]bool{})
	gob.Register(Toggles{})
	gob.Register(&ApplicationConfig{})
	gob.Register([]*MessageForUser{})
}

// KeyAndEntity is a specific pair wrapper.
type KeyAndEntity struct {
	Key    *datastore.Key
	Entity interface{}
}

type pair struct {
	First  interface{}
	Second interface{}
}

func (a *MemcacheDatastoreAccessor) recacheIdiom(c context.Context, datastoreKey *datastore.Key, idiom *Idiom) error {
	cacheKey := fmt.Sprintf("getIdiom(%v)", idiom.Id)
	err := a.cacheKeyValue(c, cacheKey, datastoreKey, idiom, 24*time.Hour)
	if err != nil {
		log.Errorf(c, err.Error())
		return err
	}

	// Batch memcache set call
	N := len(idiom.Implementations)
	cacheKeys := make([]string, N)
	for i, impl := range idiom.Implementations {
		cacheKeys[i] = fmt.Sprintf("getIdiomByImplID(%v)", impl.Id)
	}
	err = a.cacheKeysSameValue(c, cacheKeys, datastoreKey, idiom, 24*time.Hour)
	if err != nil {
		log.Errorf(c, err.Error())
		return err
	}
	// Unfortunately, some previous "getIdiomByImplID(xyz)" might be left uninvalidated.
	// (theoretically)
	return err
}

func (a *MemcacheDatastoreAccessor) uncacheIdiom(c context.Context, idiom *Idiom) error {
	cacheKeys := make([]string, 1+len(idiom.Implementations))
	cacheKeys[0] = fmt.Sprintf("getIdiom(%v)", idiom.Id)
	for i, impl := range idiom.Implementations {
		cacheKeys[1+i] = fmt.Sprintf("getIdiomByImplID(%v)", impl.Id)
	}

	err := memcache.DeleteMulti(c, cacheKeys)
	if err != nil {
		log.Errorf(c, err.Error())
	}
	return err
}

func (a *MemcacheDatastoreAccessor) getIdiom(c context.Context, idiomID int) (*datastore.Key, *Idiom, error) {
	cacheKey := fmt.Sprintf("getIdiom(%v)", idiomID)
	data, cacheerr := a.readCache(c, cacheKey)
	if cacheerr != nil {
		log.Errorf(c, cacheerr.Error())
		// Ouch. Well, skip the cache if it's broken
		return a.dataAccessor.getIdiom(c, idiomID)
	}
{'fraction_non_alphanumeric': 0.13648332358104154, 'fraction_numerical': 0.001243417203042715, 'mean_word_length': 4.580816326530612, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1543031', 'n_tokens_mistral': 4821, 'n_tokens_neox': 4640, 'n_words': 1260}