<commit_before>package script

import (
	"bytes"
	"os"
	"text/template"

	"github.com/spf13/cobra"
)

type scriptConfig struct {
	DBName      string
	BucketName  string
	LogPrefix   string
	ClientPort  string
	ProjectName string
	KeyPath     string
	Conns       int
	Clients     int
	ValSize     int
	Total       int
}

var (
	Command = &cobra.Command{
		Use:        "script",
		Short:      "Generates cloud provisioning script.",
		SuggestFor: []string{"scrt"},
		RunE:       scriptCommandFunc,
	}

	outputPath string
	cfg        = scriptConfig{}
)

func init() {
	Command.PersistentFlags().StringVarP(&outputPath, "output", "o", "script.sh", "File path to store script.")
	Command.PersistentFlags().StringVarP(&cfg.DBName, "db-name", "d", "etcdv3", "Name of database (etcdv2, etcdv3, zookeeper, zk, consul).")
	Command.PersistentFlags().StringVarP(&cfg.BucketName, "bucket-name", "b", "", "Name of bucket to store results.")
	Command.PersistentFlags().StringVarP(&cfg.LogPrefix, "log-prefix", "p", "bench-01", "Prefix to name instances, logs files.")
	Command.PersistentFlags().StringVarP(&cfg.ClientPort, "client-port", "c", "2379", "2379 for etcd, 2181 for Zookeeper, 8500 for Consul.")
	Command.PersistentFlags().StringVarP(&cfg.ProjectName, "project-name", "n", "etcd-development", "Project name.")
	Command.PersistentFlags().StringVarP(&cfg.KeyPath, "key-path", "k", "$HOME/key.json", "Key path.")
	Command.PersistentFlags().IntVar(&cfg.Conns, "conns", 1, "conns.")
	Command.PersistentFlags().IntVar(&cfg.Clients, "clients", 1, "clients.")
	Command.PersistentFlags().IntVar(&cfg.ValSize, "val-size", 256, "val-size.")
	Command.PersistentFlags().IntVar(&cfg.Total, "total", 3000000, "total.")
}

func scriptCommandFunc(cmd *cobra.Command, args []string) error {
	tpl := template.Must(template.New("scriptTemplate").Parse(scriptTemplate))
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, cfg); err != nil {
		return err
	}
	return toFile(buf.String(), outputPath)
}

const scriptTemplate = `
#!/usr/bin/env bash
set -e

gcloud compute instances list
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-1 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-2 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-3 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-tester --custom-cpu=16 --custom-memory=30 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --zone us-central1-a
gcloud compute instances list

gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-1
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-2
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-3
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-tester


#########
# agent #
#########
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
go get -v -u -f github.com/coreos/dbtester
curl https://storage.googleapis.com/etcd/dbtester_agent.sh | bash -s f /mnt/ssd0

cd /mnt/ssd0
ls /mnt/ssd0
cat /mnt/ssd0/agent.log


##########
# tester #
##########
ulimit -n 3000
ulimit -n
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
psn ps-kill --force -s dbtester
go get -v -u -f github.com/coreos/dbtester

# start test
AGENT_ENDPOINTS='___IP_ADDR_1___:3500,___IP_ADDR_2___:3500,___IP_ADDR_3___:3500'
DATABASE_ENDPOINTS='___IP_ADDR_1___:{{.ClientPort}},___IP_ADDR_2___:{{.ClientPort}},___IP_ADDR_3___:{{.ClientPort}}'

# start database
dbtester start --agent-endpoints=$(echo $AGENT_ENDPOINTS) --database={{.DBName}} --database-log-path=database.log --log-prefix={{.LogPrefix}}-{{.DBName}} --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --monitor-result-path=monitor.csv;

cat /mnt/ssd0/agent.log
cat /mnt/ssd0/database.log

# start benchmark
nohup dbtester bench --database={{.DBName}} --sample --no-histogram --csv-result-path={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --endpoints=$DATABASE_ENDPOINTS --conns={{.Conns}} --clients={{.Clients}} put --key-size=64 --val-size={{.ValSize}} --total={{.Total}} > {{.LogPrefix}}-{{.DBName}}-result.txt 2>&1 &

cat {{.LogPrefix}}-{{.DBName}}-result.txt

# benchmark done!
# stop database to trigger uploading in remote machines
dbtester stop --agent-endpoints=$(echo $AGENT_ENDPOINTS)

dbtester upload --from={{.LogPrefix}}-{{.DBName}}-timeseries.csv --to={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}


####################
# in case of panic #
####################
dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-1-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-1-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-1-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-2-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-2-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-2-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-3-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-3-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-3-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

`
<commit_msg>script: rename<commit_after>package script

import (
	"bytes"
	"os"
	"text/template"

	"github.com/spf13/cobra"
)

type scriptConfig struct {
	DBName      string
	BucketName  string
	LogPrefix   string
	ClientPort  string
	ProjectName string
	KeyPath     string
	BenchType   string
	Conns       int
	Clients     int
	ValSize     int
	Total       int
}

var (
	Command = &cobra.Command{
		Use:        "script",
		Short:      "Generates cloud provisioning script.",
		SuggestFor: []string{"scrt"},
		RunE:       scriptCommandFunc,
	}

	outputPath string
	cfg        = scriptConfig{}
)

func init() {
	Command.PersistentFlags().StringVarP(&outputPath, "output", "o", "script.sh", "File path to store script.")
	Command.PersistentFlags().StringVarP(&cfg.DBName, "db-name", "d", "etcdv3", "Name of database (etcdv2, etcdv3, zookeeper, zk, consul).")
	Command.PersistentFlags().StringVarP(&cfg.BucketName, "bucket-name", "b", "", "Name of bucket to store results.")
	Command.PersistentFlags().StringVarP(&cfg.LogPrefix, "log-prefix", "p", "bench-01", "Prefix to name instances, logs files.")
	Command.PersistentFlags().StringVarP(&cfg.ClientPort, "client-port", "c", "2379", "2379 for etcd, 2181 for Zookeeper, 8500 for Consul.")
	Command.PersistentFlags().StringVarP(&cfg.ProjectName, "project-name", "n", "etcd-development", "Project name.")
	Command.PersistentFlags().StringVarP(&cfg.KeyPath, "key-path", "k", "$HOME/key.json", "Key path.")
	Command.PersistentFlags().StringVarP(&cfg.BenchType, "bench-type", "t", "put", "'put' or 'range'.")
	Command.PersistentFlags().IntVar(&cfg.Conns, "conns", 1, "conns.")
	Command.PersistentFlags().IntVar(&cfg.Clients, "clients", 1, "clients.")
	Command.PersistentFlags().IntVar(&cfg.ValSize, "val-size", 256, "val-size.")
	Command.PersistentFlags().IntVar(&cfg.Total, "total", 3000000, "total.")
}

func scriptCommandFunc(cmd *cobra.Command, args []string) error {
	if cfg.BenchType == "range" {
		cfg.BenchType = "range --consistency s --single-key"
	}
	tpl := template.Must(template.New("scriptTemplate").Parse(scriptTemplate))
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, cfg); err != nil {
		return err
	}
	return toFile(buf.String(), outputPath)
}

const scriptTemplate = `
#!/usr/bin/env bash
set -e

gcloud compute instances list
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-1 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-2 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-3 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-tester --custom-cpu=16 --custom-memory=30 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --zone us-central1-a
gcloud compute instances list

gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-1
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-2
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-3
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-tester


#########
# agent #
#########
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
go get -v -u -f github.com/coreos/dbtester
curl https://storage.googleapis.com/etcd/dbtester_agent.sh | bash -s f /mnt/ssd0

cd /mnt/ssd0
ls /mnt/ssd0
cat /mnt/ssd0/agent.log


##########
# tester #
##########
ulimit -n 3000
ulimit -n
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
psn ps-kill --force -s dbtester
go get -v -u -f github.com/coreos/dbtester

# start test
AGENT_ENDPOINTS='___IP_ADDR_1___:3500,___IP_ADDR_2___:3500,___IP_ADDR_3___:3500'
DATABASE_ENDPOINTS='___IP_ADDR_1___:{{.ClientPort}},___IP_ADDR_2___:{{.ClientPort}},___IP_ADDR_3___:{{.ClientPort}}'

# start database
dbtester start --agent-endpoints=$(echo $AGENT_ENDPOINTS) --database={{.DBName}} --database-log-path=database.log --log-prefix={{.LogPrefix}}-{{.DBName}} --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --monitor-result-path=monitor.csv;

cat /mnt/ssd0/agent.log
cat /mnt/ssd0/database.log

# start benchmark
nohup dbtester bench --database={{.DBName}} --sample --no-histogram --csv-result-path={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --endpoints=$DATABASE_ENDPOINTS --conns={{.Conns}} --clients={{.Clients}} {{.BenchType}} --key-size=64 --val-size={{.ValSize}} --total={{.Total}} > {{.LogPrefix}}-{{.DBName}}-result.txt 2>&1 &

cat {{.LogPrefix}}-{{.DBName}}-result.txt

# benchmark done!
# stop database to trigger uploading in remote machines
dbtester stop --agent-endpoints=$(echo $AGENT_ENDPOINTS)

dbtester upload --from={{.LogPrefix}}-{{.DBName}}-timeseries.csv --to={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}


####################
# in case of panic #
####################
dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-1-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-1-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-1-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-2-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-2-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-2-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-3-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-3-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-3-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

`

{'fraction_non_alphanumeric': 0.22541683077327032, 'fraction_numerical': 0.018379939608769855, 'mean_word_length': 4.47627606038821, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5584172', 'n_tokens_mistral': 5728, 'n_tokens_neox': 5496, 'n_words': 982}