<commit_before>package xormrediscache

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/garyburd/redigo/redis"
	//"github.com/go-xorm/core"
	"hash/crc32"
	"log"
	"reflect"
	// "strconv"
	"time"
	"unsafe"
)

const (
	DEFAULT_EXPIRATION = time.Duration(0)
	FOREVER_EXPIRATION = time.Duration(-1)
)

// Wraps the Redis client to meet the Cache interface.
type RedisCacher struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCacher(host string, password string, defaultExpiration time.Duration) *RedisCacher {
	var pool = &redis.Pool{
		MaxIdle:     5,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			// the redis protocol should probably be made sett-able
			c, err := redis.Dial("tcp", host)
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err := c.Do("PING"); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			if _, err := c.Do("PING"); err != nil {
				return err
			}
			return nil
		},
	}
	return &RedisCacher{pool, defaultExpiration}
}

func exists(conn redis.Conn, key string) bool {
	existed, _ := redis.Bool(conn.Do("EXISTS", key))
	return existed
}

func (c *RedisCacher) getBeanKey(tableName string, id string) string {

	beanKey := fmt.Sprintf("bean:%s:%s", tableName, id)
	log.Printf("[xorm/redis_cacher] getBeanKey: [%s]", beanKey)
	return beanKey
}

func (c *RedisCacher) getSqlKey(tableName string, sql string) string {
	// hash sql to minimize key length
	crc := crc32.ChecksumIEEE([]byte(sql))

	sqlKey := fmt.Sprintf("sql:%s:%d", tableName, crc)
	log.Printf("[xorm/redis_cacher] getSqlKey: [%s]", sqlKey)
	return sqlKey
}

func (c *RedisCacher) Flush() error {
	conn := c.pool.Get()
	defer conn.Close()
	_, err := conn.Do("FLUSHALL")
	return err
}

func (c *RedisCacher) getObject(key string) interface{} {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return nil
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		log.Fatalf("[xorm/redis_cacher] redis.Bytes failed: %s", err)
		return nil
	}

	value, err := deserialize(item)

	return value
}

func (c *RedisCacher) getObject2(key string, ptr interface{}) error {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return err
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		log.Fatalf("[xorm/redis_cacher] redis.Bytes failed: %s", err)
		return err
	}
	err = deserialize2(item, ptr)
	return err
}

func (c *RedisCacher) GetIds(tableName, sql string) interface{} {
	log.Printf("[xorm/redis_cacher] GetIds|tableName:%s|sql:%s", tableName, sql)

	return c.getObject(c.getSqlKey(tableName, sql))
}

func (c *RedisCacher) GetBean(tableName string, id string) interface{} {
	log.Printf("[xorm/redis_cacher] GetBean|tableName:%s|id:%s", tableName, id)
	return c.getObject(c.getBeanKey(tableName, id))
}

func (c *RedisCacher) GetBean2(tableName string, id string, ptr interface{}) error {
	log.Printf("[xorm/redis_cacher] GetBean|tableName:%s|id:%s", tableName, id)
	return c.getObject2(c.getBeanKey(tableName, id), ptr)
}

func (c *RedisCacher) putObject(key string, value interface{}) {
	c.invoke(c.pool.Get().Do, key, value, c.defaultExpiration)
}

func (c *RedisCacher) PutIds(tableName, sql string, ids interface{}) {
	log.Printf("[xorm/redis_cacher] PutIds|tableName:%s|sql:%s|type:%v", tableName, sql, reflect.TypeOf(ids))

	c.putObject(c.getSqlKey(tableName, sql), ids)
}

func (c *RedisCacher) PutBean(tableName string, id string, obj interface{}) {
	log.Printf("[xorm/redis_cacher] PutBean|tableName:%s|id:%s|type:%v", tableName, id, reflect.TypeOf(obj))
	c.putObject(c.getBeanKey(tableName, id), obj)
}

func (c *RedisCacher) delObject(key string) {
	conn := c.pool.Get()
	defer conn.Close()
	if !exists(conn, key) {
		return // core.ErrCacheMiss
	}
	conn.Do("DEL", key)

	// _, err := conn.Do("DEL", key)
	// return err
}

func (c *RedisCacher) DelIds(tableName, sql string) {
	c.delObject(c.getSqlKey(tableName, sql))
}

func (c *RedisCacher) DelBean(tableName string, id string) {
	c.delObject(c.getBeanKey(tableName, id))
}

func (c *RedisCacher) clearObjects(key string) {
	conn := c.pool.Get()
	defer conn.Close()
	if exists(conn, key) {
		// _, err := conn.Do("DEL", key)
		// return err
		conn.Do("DEL", key)
	} else {
		// return ErrCacheMiss
	}
}

func (c *RedisCacher) ClearIds(tableName string) {
	c.clearObjects(c.getSqlKey(tableName, "*"))
}

func (c *RedisCacher) ClearBeans(tableName string) {
	c.clearObjects(c.getBeanKey(tableName, "*"))
}

func (c *RedisCacher) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DEFAULT_EXPIRATION:
		expires = c.defaultExpiration
	case FOREVER_EXPIRATION:
		expires = time.Duration(0)
	}

	b, err := serialize(value)
	if err != nil {
		return err
	}<commit_msg>code tidy up<commit_after>package xormrediscache

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/go-xorm/core"
	"hash/crc32"
	"log"
	"reflect"
	// "strconv"
	"time"
	"unsafe"
)

const (
	DEFAULT_EXPIRATION = time.Duration(0)
	FOREVER_EXPIRATION = time.Duration(-1)
)

// Wraps the Redis client to meet the Cache interface.
type RedisCacher struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCacher(host string, password string, defaultExpiration time.Duration) *RedisCacher {
	var pool = &redis.Pool{
		MaxIdle:     5,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			// the redis protocol should probably be made sett-able
			c, err := redis.Dial("tcp", host)
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err := c.Do("PING"); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			if _, err := c.Do("PING"); err != nil {
				return err
			}
			return nil
		},
	}
	return &RedisCacher{pool, defaultExpiration}
}

func exists(conn redis.Conn, key string) bool {
	existed, _ := redis.Bool(conn.Do("EXISTS", key))
	return existed
}

func (c *RedisCacher) getBeanKey(tableName string, id string) string {

	beanKey := fmt.Sprintf("xorm:bean:%s:%s", tableName, id)
	log.Printf("[xorm/redis_cacher] getBeanKey: [%s]", beanKey)
	return beanKey
}

func (c *RedisCacher) getSqlKey(tableName string, sql string) string {
	// hash sql to minimize key length
	crc := crc32.ChecksumIEEE([]byte(sql))

	sqlKey := fmt.Sprintf("xorm:sql:%s:%d", tableName, crc)
	log.Printf("[xorm/redis_cacher] getSqlKey: [%s]", sqlKey)
	return sqlKey
}

func (c *RedisCacher) Flush() error {
	// conn := c.pool.Get()
	// defer conn.Close()
	// _, err := conn.Do("FLUSHALL")
	// return err
	return c.delObject("xorm:*")
}

func (c *RedisCacher) getObject(key string) interface{} {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return nil
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		log.Fatalf("[xorm/redis_cacher] redis.Bytes failed: %s", err)
		return nil
	}

	value, err := deserialize(item)

	return value
}

func (c *RedisCacher) getObject2(key string, ptr interface{}) error {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return err
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		log.Fatalf("[xorm/redis_cacher] redis.Bytes failed: %s", err)
		return err
	}
	err = deserialize2(item, ptr)
	return err
}

func (c *RedisCacher) GetIds(tableName, sql string) interface{} {
	log.Printf("[xorm/redis_cacher] GetIds|tableName:%s|sql:%s", tableName, sql)

	return c.getObject(c.getSqlKey(tableName, sql))
}

func (c *RedisCacher) GetBean(tableName string, id string) interface{} {
	log.Printf("[xorm/redis_cacher] GetBean|tableName:%s|id:%s", tableName, id)
	return c.getObject(c.getBeanKey(tableName, id))
}

func (c *RedisCacher) GetBean2(tableName string, id string, ptr interface{}) error {
	log.Printf("[xorm/redis_cacher] GetBean|tableName:%s|id:%s", tableName, id)
	return c.getObject2(c.getBeanKey(tableName, id), ptr)
}

func (c *RedisCacher) putObject(key string, value interface{}) {
	c.invoke(c.pool.Get().Do, key, value, c.defaultExpiration)
}

func (c *RedisCacher) PutIds(tableName, sql string, ids interface{}) {
	log.Printf("[xorm/redis_cacher] PutIds|tableName:%s|sql:%s|type:%v", tableName, sql, reflect.TypeOf(ids))

	c.putObject(c.getSqlKey(tableName, sql), ids)
}

func (c *RedisCacher) PutBean(tableName string, id string, obj interface{}) {
	log.Printf("[xorm/redis_cacher] PutBean|tableName:%s|id:%s|type:%v", tableName, id, reflect.TypeOf(obj))
	c.putObject(c.getBeanKey(tableName, id), obj)
}

func (c *RedisCacher) delObject(key string) error {
	log.Printf("[xorm/redis_cacher] delObject key:[%s]", key)

	conn := c.pool.Get()
	defer conn.Close()
	if !exists(conn, key) {
		log.Printf("[xorm/redis_cacher] delObject: %v", core.ErrCacheMiss)
		return core.ErrCacheMiss
	}
	_, err := conn.Do("DEL", key)
	return err
}

func (c *RedisCacher) delObjects(key string) error {

	log.Printf("[xorm/redis_cacher] delObjects key:[%s]", key)

	conn := c.pool.Get()
	defer conn.Close()

	keys, err := conn.Do("KEYS", key)
	log.Printf("[xorm/redis_cacher] delObjects keys: %v", keys)

	if err == nil {
		for _, key := range keys.([]interface{}) {
			conn.Do("DEL", key)
		}
	}
	return err
}

func (c *RedisCacher) DelIds(tableName, sql string) {
	c.delObject(c.getSqlKey(tableName, sql))
}

func (c *RedisCacher) DelBean(tableName string, id string) {
	c.delObject(c.getBeanKey(tableName, id))
}

func (c *RedisCacher) ClearIds(tableName string) {
	c.delObjects(c.getSqlKey(tableName, "*"))
}

func (c *RedisCacher) ClearBeans(tableName string) {
	c.delObjects(c.getBeanKey(tableName, "*"))
}

func (c *RedisCacher) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DEFAULT_EXPIRATION:
		expires = c.defaultExpiration
	case FOREVER_EXPIRATION:
		expires = time.Duration(0)
	}

	b, err := serialize(value)
	if err != nil {
		return err
	}
{'fraction_non_alphanumeric': 0.15878189607003165, 'fraction_numerical': 0.002980070776680946, 'mean_word_length': 4.190430159497342, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462731', 'n_tokens_mistral': 4296, 'n_tokens_neox': 4107, 'n_words': 977}