<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sequencer

import (
	"context"
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/empty"
	"github.com/google/trillian/monitoring"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/google/keytransparency/core/directory"
	"github.com/google/keytransparency/core/mutator"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/keytransparency/core/sequencer/mapper"
	"github.com/google/keytransparency/core/sequencer/metadata"
	"github.com/google/keytransparency/core/sequencer/runner"
	"github.com/google/keytransparency/core/water"

	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
	tpb "github.com/google/trillian"
)

const (
	directoryIDLabel = "directoryid"
	logIDLabel       = "logid"
	reasonLabel      = "reason"
	fnLabel          = "fn"
)

var (
	initMetrics        sync.Once
	knownDirectories   monitoring.Gauge
	logEntryCount      monitoring.Counter
	logEntryUnapplied  monitoring.Gauge
	mapLeafCount       monitoring.Counter
	fnCount            monitoring.Counter
	mapRevisionCount   monitoring.Counter
	watermarkWritten   monitoring.Gauge
	watermarkDefined   monitoring.Gauge
	watermarkApplied   monitoring.Gauge
	appliedLatency     monitoring.Histogram
	mutationFailures   monitoring.Counter
	fnLatency          monitoring.Histogram
	logRootTrail       monitoring.Gauge
	unappliedRevisions monitoring.Gauge
)

func createMetrics(mf monitoring.MetricFactory) {
	knownDirectories = mf.NewGauge(
		"known_directories",
		"Set to 1 for known directories (whether this instance is master or not)",
		directoryIDLabel)
	logEntryCount = mf.NewCounter(
		"log_entry_count",
		"Total number of log entries read since process start. Duplicates are not removed.",
		directoryIDLabel, logIDLabel)
	logEntryUnapplied = mf.NewGauge(
		"log_entry_unapplied",
		"Total number of log entries still to be processed in the queue.",
		directoryIDLabel)
	mapLeafCount = mf.NewCounter(
		"map_leaf_count",
		"Total number of map leaves written since process start. Duplicates are not removed.",
		directoryIDLabel)
	fnCount = mf.NewCounter(
		"fn_count",
		"Total number of mapping operations that have run since process start",
		directoryIDLabel, fnLabel)
	mapRevisionCount = mf.NewCounter(
		"map_revision_count",
		"Total number of map revisions written since process start.",
		directoryIDLabel)
	watermarkWritten = mf.NewGauge(
		"watermark_written",
		"High watermark of each input log that has been written",
		directoryIDLabel, logIDLabel)
	watermarkDefined = mf.NewGauge(
		"watermark_defined",
		"High watermark of each input log that has been defined in the batch table",
		directoryIDLabel, logIDLabel)
	watermarkApplied = mf.NewGauge(
		"watermark_applied",
		"High watermark of each input log that has been committed in a map revision",
		directoryIDLabel, logIDLabel)
	appliedLatency = mf.NewHistogram(
		"applied_latency",
		"Latency between creating a mutation entry and putting it to a map revision, in seconds",
		directoryIDLabel, logIDLabel)
	mutationFailures = mf.NewCounter(
		"mutation_failures",
		"Number of invalid mutations the signer has processed for directoryid since process start",
		directoryIDLabel, reasonLabel)
	fnLatency = mf.NewHistogram(
		"apply_revision_latency",
		"Latency of sequencer apply revision operation in seconds",
		directoryIDLabel, fnLabel)
	logRootTrail = mf.NewGauge(
		"log_root_trail",
		"How many revisions have not been published to the log",
	)
	unappliedRevisions = mf.NewGauge(
		"unapplied_revisions",
		"How many revisions have been defined but haven't been applied to the map",<commit_msg>Specify buckets for latency distribution (#1414)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sequencer

import (
	"context"
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/empty"
	"github.com/google/trillian/monitoring"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/google/keytransparency/core/directory"
	"github.com/google/keytransparency/core/mutator"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/keytransparency/core/sequencer/mapper"
	"github.com/google/keytransparency/core/sequencer/metadata"
	"github.com/google/keytransparency/core/sequencer/runner"
	"github.com/google/keytransparency/core/water"

	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
	tpb "github.com/google/trillian"
)

const (
	directoryIDLabel = "directoryid"
	logIDLabel       = "logid"
	reasonLabel      = "reason"
	fnLabel          = "fn"
)

var (
	initMetrics        sync.Once
	knownDirectories   monitoring.Gauge
	logEntryCount      monitoring.Counter
	logEntryUnapplied  monitoring.Gauge
	mapLeafCount       monitoring.Counter
	fnCount            monitoring.Counter
	mapRevisionCount   monitoring.Counter
	watermarkWritten   monitoring.Gauge
	watermarkDefined   monitoring.Gauge
	watermarkApplied   monitoring.Gauge
	appliedLatency     monitoring.Histogram
	mutationFailures   monitoring.Counter
	fnLatency          monitoring.Histogram
	logRootTrail       monitoring.Gauge
	unappliedRevisions monitoring.Gauge
)

func createMetrics(mf monitoring.MetricFactory) {
	knownDirectories = mf.NewGauge(
		"known_directories",
		"Set to 1 for known directories (whether this instance is master or not)",
		directoryIDLabel)
	logEntryCount = mf.NewCounter(
		"log_entry_count",
		"Total number of log entries read since process start. Duplicates are not removed.",
		directoryIDLabel, logIDLabel)
	logEntryUnapplied = mf.NewGauge(
		"log_entry_unapplied",
		"Total number of log entries still to be processed in the queue.",
		directoryIDLabel)
	mapLeafCount = mf.NewCounter(
		"map_leaf_count",
		"Total number of map leaves written since process start. Duplicates are not removed.",
		directoryIDLabel)
	fnCount = mf.NewCounter(
		"fn_count",
		"Total number of mapping operations that have run since process start",
		directoryIDLabel, fnLabel)
	mapRevisionCount = mf.NewCounter(
		"map_revision_count",
		"Total number of map revisions written since process start.",
		directoryIDLabel)
	watermarkWritten = mf.NewGauge(
		"watermark_written",
		"High watermark of each input log that has been written",
		directoryIDLabel, logIDLabel)
	watermarkDefined = mf.NewGauge(
		"watermark_defined",
		"High watermark of each input log that has been defined in the batch table",
		directoryIDLabel, logIDLabel)
	watermarkApplied = mf.NewGauge(
		"watermark_applied",
		"High watermark of each input log that has been committed in a map revision",
		directoryIDLabel, logIDLabel)
	appliedLatency = mf.NewHistogramWithBuckets(
		"applied_latency",
		"Latency between creating a mutation entry and putting it to a map revision, in seconds",
		monitoring.LatencyBuckets(),
		directoryIDLabel, logIDLabel)
	mutationFailures = mf.NewCounter(
		"mutation_failures",
		"Number of invalid mutations the signer has processed for directoryid since process start",
		directoryIDLabel, reasonLabel)
	fnLatency = mf.NewHistogram(
		"apply_revision_latency",
		"Latency of sequencer apply revision operation in seconds",
		directoryIDLabel, fnLabel)
	logRootTrail = mf.NewGauge(
		"log_root_trail",
		"How many revisions have not been published to the log",
	)
	unappliedRevisions = mf.NewGauge(
		"unapplied_revisions",
		"How many revisions have been defined but haven't been applied to the map",
{'fraction_non_alphanumeric': 0.08360898825139729, 'fraction_numerical': 0.002509410288582183, 'mean_word_length': 4.459526774595267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '737803', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2689, 'n_words': 775}