<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// maphammer is a stress/load test for a Trillian Map.
package main

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/google/trillian/testonly/hammer"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"

	_ "github.com/google/trillian/merkle/coniks"    // register CONIKS_SHA512_256
	_ "github.com/google/trillian/merkle/maphasher" // register TEST_MAP_HASHER
)

var (
	mapIDs          = flag.String("map_ids", "", "Comma-separated list of map IDs to test; ephemeral tree used if empty")
	rpcServer       = flag.String("rpc_server", "", "Server address:port")
	adminServer     = flag.String("admin_server", "", "Address of the gRPC Trillian Admin Server (host:port)")
	metricsEndpoint = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will not be exposed")
	outLog          = flag.String("log_to", "", "File to record operations in")
	seed            = flag.Int64("seed", -1, "Seed for random number generation")
	operations      = flag.Uint64("operations", ^uint64(0), "Number of operations to perform")
	minLeaves       = flag.Int("min_leaves", 0, "Minimum count of leaves to affect per-operation")
	maxLeaves       = flag.Int("max_leaves", 10, "Maximum count of leaves to affect per-operation")
	leafSize        = flag.Uint("leaf_size", 100, "Size of leaf values")
	extraSize       = flag.Uint("extra_size", 100, "Size of leaf extra data")
	checkers        = flag.Int("checkers", 0, "Number of checker goroutines to run")
	retryErrors     = flag.Bool("retry_errors", false, "Whether to retry failed operations")
	opDeadline      = flag.Duration("op_deadline", 60*time.Second, "How long to wait for operation success")
	emitInterval    = flag.Duration("emit_interval", 0, "How often to output the Hammer state")
)
var (
	getLeavesBias    = flag.Int("get_leaves", 20, "Bias for get-leaves operations")
	getLeavesRevBias = flag.Int("get_leaves_rev", 2, "Bias for get-leaves-revision operations")
	setLeavesBias    = flag.Int("set_leaves", 20, "Bias for set-leaves operations")
	getSMRBias       = flag.Int("get_smr", 10, "Bias for get-smr operations")
	getSMRRevBias    = flag.Int("get_smr_rev", 2, "Bias for get-smr-revision operations")
	invalidChance    = flag.Int("invalid_chance", 10, "Chance of generating an invalid operation, as the N in 1-in-N (0 for never)")
)

func hammerTime() {
	// Purely for fun, so no error checking
	fmt.Print("\n\nIf they'd let me have my way I could have flayed him into shape.\n")
	for i := 0; i < 8; i++ {
		time.Sleep(100 * time.Millisecond)
		fmt.Print(".")
	}
	fmt.Print("\n\n")
	mc := "<KEY>"
	mcData, _ := base64.StdEncoding.DecodeString(mc)
	b := bytes.NewReader(mcData)
	r, _ := gzip.NewReader(b)
	io.Copy(os.Stdout, r)
	r.Close()
	fmt.Print("\n\nLet me hammer him today?\n\n")
}

func main() {
	flag.Parse()
	defer glog.Flush()

	if *mapIDs == "" {
		glog.Info("No mapIDs provided so using a transient tree")
		*mapIDs = "0"
	}
	if *seed == -1 {
		*seed = time.Now().UTC().UnixNano() & 0xFFFFFFFF
	}
	fmt.Printf("Today's test has been brought to you by the letters M, A, and P and the number %#x\n", *seed)

	bias := hammer.MapBias{
		Bias: map[hammer.MapEntrypointName]int{
			hammer.GetLeavesName:    *getLeavesBias,
			hammer.GetLeavesRevName: *getLeavesRevBias,
			hammer.SetLeavesName:    *setLeavesBias,
			hammer.GetSMRName:       *getSMRBias,
			hammer.GetSMRRevName:    *getSMRRevBias,
		},
		InvalidChance: map[hammer.MapEntrypointName]int{
			hammer.GetLeavesName:    *invalidChance,
			hammer.GetLeavesRevName: *invalidChance,
			hammer.SetLeavesName:    *invalidChance,
			hammer.GetSMRName:       0,
			hammer.GetSMRRevName:    *getSMRRevBias,
		},
	}

	var mf monitoring.MetricFactory
	if *metricsEndpoint != "" {
		mf = prometheus.MetricFactory{}
		http.Handle("/metrics", promhttp.Handler())
		server := http.Server{Addr: *metricsEndpoint, Handler: nil}
		glog.Infof("Serving metrics at %v", *metricsEndpoint)
		go func() {
			err := server.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		mf = monitoring.InertMetricFactory{}
	}

	if glog.V(1) {
		hammerTime()<commit_msg>maphammer: fix incorrect GetSMR@Rev bias<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// maphammer is a stress/load test for a Trillian Map.
package main

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/google/trillian/testonly/hammer"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"

	_ "github.com/google/trillian/merkle/coniks"    // register CONIKS_SHA512_256
	_ "github.com/google/trillian/merkle/maphasher" // register TEST_MAP_HASHER
)

var (
	mapIDs          = flag.String("map_ids", "", "Comma-separated list of map IDs to test; ephemeral tree used if empty")
	rpcServer       = flag.String("rpc_server", "", "Server address:port")
	adminServer     = flag.String("admin_server", "", "Address of the gRPC Trillian Admin Server (host:port)")
	metricsEndpoint = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will not be exposed")
	outLog          = flag.String("log_to", "", "File to record operations in")
	seed            = flag.Int64("seed", -1, "Seed for random number generation")
	operations      = flag.Uint64("operations", ^uint64(0), "Number of operations to perform")
	minLeaves       = flag.Int("min_leaves", 0, "Minimum count of leaves to affect per-operation")
	maxLeaves       = flag.Int("max_leaves", 10, "Maximum count of leaves to affect per-operation")
	leafSize        = flag.Uint("leaf_size", 100, "Size of leaf values")
	extraSize       = flag.Uint("extra_size", 100, "Size of leaf extra data")
	checkers        = flag.Int("checkers", 0, "Number of checker goroutines to run")
	retryErrors     = flag.Bool("retry_errors", false, "Whether to retry failed operations")
	opDeadline      = flag.Duration("op_deadline", 60*time.Second, "How long to wait for operation success")
	emitInterval    = flag.Duration("emit_interval", 0, "How often to output the Hammer state")
)
var (
	getLeavesBias    = flag.Int("get_leaves", 20, "Bias for get-leaves operations")
	getLeavesRevBias = flag.Int("get_leaves_rev", 2, "Bias for get-leaves-revision operations")
	setLeavesBias    = flag.Int("set_leaves", 20, "Bias for set-leaves operations")
	getSMRBias       = flag.Int("get_smr", 10, "Bias for get-smr operations")
	getSMRRevBias    = flag.Int("get_smr_rev", 2, "Bias for get-smr-revision operations")
	invalidChance    = flag.Int("invalid_chance", 10, "Chance of generating an invalid operation, as the N in 1-in-N (0 for never)")
)

func hammerTime() {
	// Purely for fun, so no error checking
	fmt.Print("\n\nIf they'd let me have my way I could have flayed him into shape.\n")
	for i := 0; i < 8; i++ {
		time.Sleep(100 * time.Millisecond)
		fmt.Print(".")
	}
	fmt.Print("\n\n")
	mc := "<KEY>"
	mcData, _ := base64.StdEncoding.DecodeString(mc)
	b := bytes.NewReader(mcData)
	r, _ := gzip.NewReader(b)
	io.Copy(os.Stdout, r)
	r.Close()
	fmt.Print("\n\nLet me hammer him today?\n\n")
}

func main() {
	flag.Parse()
	defer glog.Flush()

	if *mapIDs == "" {
		glog.Info("No mapIDs provided so using a transient tree")
		*mapIDs = "0"
	}
	if *seed == -1 {
		*seed = time.Now().UTC().UnixNano() & 0xFFFFFFFF
	}
	fmt.Printf("Today's test has been brought to you by the letters M, A, and P and the number %#x\n", *seed)

	bias := hammer.MapBias{
		Bias: map[hammer.MapEntrypointName]int{
			hammer.GetLeavesName:    *getLeavesBias,
			hammer.GetLeavesRevName: *getLeavesRevBias,
			hammer.SetLeavesName:    *setLeavesBias,
			hammer.GetSMRName:       *getSMRBias,
			hammer.GetSMRRevName:    *getSMRRevBias,
		},
		InvalidChance: map[hammer.MapEntrypointName]int{
			hammer.GetLeavesName:    *invalidChance,
			hammer.GetLeavesRevName: *invalidChance,
			hammer.SetLeavesName:    *invalidChance,
			hammer.GetSMRName:       0,
			hammer.GetSMRRevName:    *invalidChance,
		},
	}

	var mf monitoring.MetricFactory
	if *metricsEndpoint != "" {
		mf = prometheus.MetricFactory{}
		http.Handle("/metrics", promhttp.Handler())
		server := http.Server{Addr: *metricsEndpoint, Handler: nil}
		glog.Infof("Serving metrics at %v", *metricsEndpoint)
		go func() {
			err := server.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		mf = monitoring.InertMetricFactory{}
	}

	if glog.V(1) {
		hammerTime()
{'fraction_non_alphanumeric': 0.1336108633164833, 'fraction_numerical': 0.012092377837248489, 'mean_word_length': 3.872042491549976, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2471144', 'n_tokens_mistral': 3703, 'n_tokens_neox': 3427, 'n_words': 935}