<commit_before>package memcache

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	youtubeMemcache "github.com/youtube/vitess/go/memcache"
)

// Restore : to restore the memcached dump data
func Restore(address string, dialTimeout time.Duration) {
	dec := json.NewDecoder(os.Stdin)
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	for {
		var kv Kv
		if err := dec.Decode(&kv); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		var ok bool
		var err error
		if kv.Cas != 0 {
			ok, err = conn.Cas(kv.Key, kv.Flags, kv.Exptime, kv.Value, kv.Cas)
		} else {
			ok, err = conn.Set(kv.Key, kv.Flags, kv.Exptime, kv.Value)
		}
		switch {
		case err != nil:
			log.Fatalf("Set key:%s, err:%s", kv.Key, err)
		case !ok:
			log.Fatalf("not stored :Set key:%s", kv.Key)
		}
		fmt.Printf("store: %#v\n", kv)
	}
}

// Stats : get stats
func Stats(address string, dialTimeout time.Duration, params string) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	result, _ := conn.Stats(params)
	fmt.Printf("%s", result)
}

// List : list all keys
func List(address string, dialTimeout time.Duration) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	keyCh := getListKeysChan(conn)
	for key := range keyCh {
		fmt.Println(key)
	}
}

// Dump all data
func Dump(address string, dialTimeout time.Duration) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	keyCh := getListKeysChan(conn)
	getConn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	for key := range keyCh {
		results, err := getConn.Get(key.name)
		if err != nil {
			log.Fatal(err)
		}
		for _, ret := range results {
			b, err := json.Marshal(Kv{ret.Key, ret.Flags, key.exptime, ret.Cas, ret.Value})
			if err != nil {
				log.Fatal(err)
			}
			fmt.Printf("%s\n", b)
		}
	}
}

// Kv : json
type Kv struct {
	// Key name
	Key string `json:"key"`
	// Flags data
	Flags uint16 `json:"flag,omitempty"`
	// Exptime expire time(second)
	Exptime uint64 `json:"exp,omitempty"`
	// Cas data
	Cas uint64 `json:"cas,omitempty"`
	// Value data
	Value []byte `json:"val"`
}

type item struct {
	key  int
	size int
}

type keyInfo struct {
	name    string
	nbytes  uint64
	exptime uint64
}

func getListKeysChan(conn *youtubeMemcache.Connection) chan keyInfo {
	keyCh := make(chan keyInfo)
	go getListKeys(conn, keyCh)
	return keyCh
}

func getListKeys(conn *youtubeMemcache.Connection, keyCh chan keyInfo) {<commit_msg>interface of<commit_after>package memcache

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	youtubeMemcache "github.com/youtube/vitess/go/memcache"
)

// Restore : to restore the memcached dump data
func Restore(address string, dialTimeout time.Duration) {
	dec := json.NewDecoder(os.Stdin)
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	for {
		var kv Kv
		if err := dec.Decode(&kv); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		var ok bool
		var err error
		if kv.Cas != 0 {
			ok, err = conn.Cas(kv.Key, kv.Flags, kv.Exptime, kv.Value, kv.Cas)
		} else {
			ok, err = conn.Set(kv.Key, kv.Flags, kv.Exptime, kv.Value)
		}
		switch {
		case err != nil:
			log.Fatalf("Set key:%s, err:%s", kv.Key, err)
		case !ok:
			log.Fatalf("not stored :Set key:%s", kv.Key)
		}
		fmt.Printf("store: %#v\n", kv)
	}
}

// Stats : get stats
func Stats(address string, dialTimeout time.Duration, params string) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	result, _ := conn.Stats(params)
	fmt.Printf("%s", result)
}

// List : list all keys
func List(address string, dialTimeout time.Duration) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	keyCh := getListKeysChan(conn)
	for key := range keyCh {
		fmt.Println(key)
	}
}

// Dump all data
func Dump(address string, dialTimeout time.Duration) {
	conn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	defer conn.Close()
	keyCh := getListKeysChan(conn)
	getConn, err := youtubeMemcache.Connect(address, dialTimeout)
	if err != nil {
		log.Fatalf("%#v", err)
	}
	for key := range keyCh {
		results, err := getConn.Get(key.name)
		if err != nil {
			log.Fatal(err)
		}
		for _, ret := range results {
			b, err := json.Marshal(Kv{ret.Key, ret.Flags, key.exptime, ret.Cas, ret.Value})
			if err != nil {
				log.Fatal(err)
			}
			fmt.Printf("%s\n", b)
		}
	}
}

// Kv : json
type Kv struct {
	// Key name
	Key string `json:"key"`
	// Flags data
	Flags uint16 `json:"flag,omitempty"`
	// Exptime expire time(second)
	Exptime uint64 `json:"exp,omitempty"`
	// Cas data
	Cas uint64 `json:"cas,omitempty"`
	// Value data
	Value []byte `json:"val"`
}

type item struct {
	key  int
	size int
}

type keyInfo struct {
	name    string
	nbytes  uint64
	exptime uint64
}

type getStats interface {
	Stats(argument string) (result []byte, err error)
}

func getListKeysChan(conn getStats) chan keyInfo {
	keyCh := make(chan keyInfo)
	go getListKeys(conn, keyCh)
	return keyCh
}

func getListKeys(conn getStats, keyCh chan keyInfo) {
{'fraction_non_alphanumeric': 0.1531953195319532, 'fraction_numerical': 0.0039603960396039604, 'mean_word_length': 3.520748576078112, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471428', 'n_tokens_mistral': 2258, 'n_tokens_neox': 2109, 'n_words': 550}