<commit_before>// Copyright 2016 <NAME>, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"log"
	"net"
	"net/http"

	"golang.org/x/net/context"

	"github.com/boltdb/bolt"
	"github.com/foolusion/elwinprotos/storage"
	"github.com/gogo/protobuf/proto"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

func bind(s []string) error {
	if len(s) == 0 {
		return nil
	}
	if err := viper.BindEnv(s[0]); err != nil {
		return err
	}
	return bind(s[1:])
}

func main() {
	log.Println("Starting bolt-store...")

	viper.SetDefault("db_file", "test.db")
	viper.SetDefault("db_bucket", "dev")
	viper.SetDefault("listen_address", ":8080")
	viper.SetDefault("metrics_address", ":8081")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/elwin/bolt-store")
	if err := viper.ReadInConfig(); err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	viper.SetEnvPrefix("bolt_store")
	if err := bind([]string{
		"db_file",
		"listen_address",
		"metrics_address",
	}); err != nil {
		log.Fatal(err)
	}

	server, err := newServer(viper.GetString("db_file"), viper.GetString("db_bucket"))

	log.Printf("lisening for grpc on %q", viper.GetString("listen_address"))
	lis, err := net.Listen("tcp", viper.GetString("listen_address"))
	if err != nil {
		log.Fatal(err)
	}

	s := grpc.NewServer(
		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
	)
	storage.RegisterElwinStorageServer(s, server)
	grpc_prometheus.Register(s)
	go func() {
		http.Handle("/metrics", prometheus.Handler())
		log.Printf("listening for /metrics on %q", viper.GetString("metrics_address"))
		log.Fatal(http.ListenAndServe(viper.GetString("metrics_address"), nil))
	}()

	log.Fatal(s.Serve(lis))
}

var (
	environmentStaging    = []byte("dev")
	environmentProduction = []byte("prod")
)

type server struct {
	db     *bolt.DB
	bucket []byte
}

func newServer(file, bucket string) (*server, error) {
	db, err := bolt.Open(file, 0600, nil)
	if err != nil {
		return nil, err
	}

	if bucket == "" {
		return nil, errors.New("bucket is empty")
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucket([]byte(bucket)); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return &server{db: db, bucket: []byte(bucket)}, nil
}

func (s *server) Close() error {
	return s.db.Close()
}

// List returns all the expriments that match a query.
func (s *server) List(ctx context.Context, r *storage.ListRequest) (*storage.ListReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}

	ar := &storage.ListReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket(s.bucket).Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var exp storage.Experiment
			if err := proto.Unmarshal(v, &exp); err != nil {
				return err
			}
			ar.Experiments = append(ar.Experiments, &exp)
		}
		return nil
	}); err != nil {
		return nil, err<commit_msg>cmd/bolt-store: filter experiments based on query<commit_after>// Copyright 2016 <NAME>, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"log"
	"net"
	"net/http"

	"k8s.io/apimachinery/pkg/labels"

	"golang.org/x/net/context"

	"github.com/boltdb/bolt"
	"github.com/foolusion/elwinprotos/storage"
	"github.com/gogo/protobuf/proto"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

func bind(s []string) error {
	if len(s) == 0 {
		return nil
	}
	if err := viper.BindEnv(s[0]); err != nil {
		return err
	}
	return bind(s[1:])
}

func main() {
	log.Println("Starting bolt-store...")

	viper.SetDefault("db_file", "test.db")
	viper.SetDefault("db_bucket", "dev")
	viper.SetDefault("listen_address", ":8080")
	viper.SetDefault("metrics_address", ":8081")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/elwin/bolt-store")
	if err := viper.ReadInConfig(); err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	viper.SetEnvPrefix("bolt_store")
	if err := bind([]string{
		"db_file",
		"listen_address",
		"metrics_address",
	}); err != nil {
		log.Fatal(err)
	}

	server, err := newServer(viper.GetString("db_file"), viper.GetString("db_bucket"))

	log.Printf("lisening for grpc on %q", viper.GetString("listen_address"))
	lis, err := net.Listen("tcp", viper.GetString("listen_address"))
	if err != nil {
		log.Fatal(err)
	}

	s := grpc.NewServer(
		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
	)
	storage.RegisterElwinStorageServer(s, server)
	grpc_prometheus.Register(s)
	go func() {
		http.Handle("/metrics", prometheus.Handler())
		log.Printf("listening for /metrics on %q", viper.GetString("metrics_address"))
		log.Fatal(http.ListenAndServe(viper.GetString("metrics_address"), nil))
	}()

	log.Fatal(s.Serve(lis))
}

var (
	environmentStaging    = []byte("dev")
	environmentProduction = []byte("prod")
)

type server struct {
	db     *bolt.DB
	bucket []byte
}

func newServer(file, bucket string) (*server, error) {
	db, err := bolt.Open(file, 0600, nil)
	if err != nil {
		return nil, err
	}

	if bucket == "" {
		return nil, errors.New("bucket is empty")
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucket([]byte(bucket)); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return &server{db: db, bucket: []byte(bucket)}, nil
}

func (s *server) Close() error {
	return s.db.Close()
}

// List returns all the expriments that match a query.
func (s *server) List(ctx context.Context, r *storage.ListRequest) (*storage.ListReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}

	selector, err := labels.Parse(r.Query)
	if err != nil {
		return nil, errors.Wrap(err, "could not parse query")
	}

	ar := &storage.ListReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket(s.bucket).Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var exp storage.Experiment
			if err := proto.Unmarshal(v, &exp); err != nil {
				return err
			}
			if selector.Matches(labels.Set(exp.Labels)) {
				ar.Experiments = append(ar.Experiments, &exp)
			}
		}
		return nil
	}); err != nil {
		return nil, err
{'fraction_non_alphanumeric': 0.15854121818412056, 'fraction_numerical': 0.006458148664049639, 'mean_word_length': 3.945522855353788, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4474309', 'n_tokens_mistral': 2991, 'n_tokens_neox': 2801, 'n_words': 696}