<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package agent

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/Godeps/_workspace/src/google.golang.org/grpc"
	"github.com/gyuho/psn/ps"
)

type (
	Flags struct {
		GRPCPort        string
		WorkingDir      string
		Monitor         bool
		MonitorInterval time.Duration
		MonitorFilePath string
	}

	// ZookeeperConfig is zookeeper configuration.
	// http://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
	ZookeeperConfig struct {
		TickTime       int
		DataDir        string
		ClientPort     string
		InitLimit      int
		SyncLimit      int
		PreAllocSize   int64
		MaxClientCnxns int64
		Peers          []ZookeeperPeer
	}
	ZookeeperPeer struct {
		MyID int
		IP   string
	}
)

var (
	shell = os.Getenv("SHELL")

	etcdBinaryPath = filepath.Join(os.Getenv("GOPATH"), "bin/etcd")
	etcdToken      = "etcd_token"

	dataLogPath  = "data.log"
	etcdDataDir  = "data.etcd"
	zkWorkingDir = "zookeeper"
	zkDataDir    = "zookeeper/data.zk"

	zkTemplate = `tickTime={{.TickTime}}
dataDir={{.DataDir}}
clientPort={{.ClientPort}}
initLimit={{.InitLimit}}
syncLimit={{.SyncLimit}}
preAllocSize={{.PreAllocSize}}
maxClientCnxns={{.MaxClientCnxns}}
{{range .Peers}}server.{{.MyID}}={{.IP}}:2888:3888
{{end}}
`
	zkConfig = ZookeeperConfig{
		TickTime:       2000,
		ClientPort:     "2181",
		InitLimit:      5,
		SyncLimit:      5,
		PreAllocSize:   65536 * 1024,
		MaxClientCnxns: 60,
		Peers: []ZookeeperPeer{
			{MyID: 1, IP: "10.240.0.12"},
			{MyID: 2, IP: "10.240.0.13"},
			{MyID: 3, IP: "10.240.0.14"},
		},
	}

	Command = &cobra.Command{
		Use:   "agent",
		Short: "Database agent in remote servers.",
		Run:   CommandFunc,
	}
	globalFlags = Flags{}
)

func homeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func init() {
	if len(shell) == 0 {
		shell = "sh"
	}
	Command.PersistentFlags().StringVarP(&globalFlags.GRPCPort, "agent-port", "p", ":3500", "Port to server agent gRPC server.")
	Command.PersistentFlags().StringVarP(&globalFlags.WorkingDir, "working-dir", "d", homeDir(), "Working directory to store data.")
	Command.PersistentFlags().BoolVar(&globalFlags.Monitor, "monitor", false, "Periodically records resource usage.")
	Command.PersistentFlags().DurationVar(&globalFlags.MonitorInterval, "monitor-interval", time.Second, "Resource monitor interval.")
	Command.PersistentFlags().StringVar(&globalFlags.MonitorFilePath, "monitor-path", "monitor.csv", "File path to store monitor results.")
}

func CommandFunc(cmd *cobra.Command, args []string) {
	dataLogPath = filepath.Join(globalFlags.WorkingDir, dataLogPath)
	etcdDataDir = filepath.Join(globalFlags.WorkingDir, etcdDataDir)
	zkWorkingDir = filepath.Join(globalFlags.WorkingDir, zkWorkingDir)
	zkDataDir = filepath.Join(globalFlags.WorkingDir, zkDataDir)

	log.Printf("gRPC has started serving at  %s\n", globalFlags.GRPCPort)
	log.Printf("data log path:               %s\n", dataLogPath)
	log.Printf("etcd data directory:         %s\n", etcdDataDir)
	log.Printf("Zookeeper working directory: %s\n", zkWorkingDir)
	log.Printf("Zookeeper data directory:    %s\n", zkDataDir)

	var (
		grpcServer = grpc.NewServer()
		sender     = NewTransporterServer()
	)
	ln, err := net.Listen("tcp", globalFlags.GRPCPort)
	if err != nil {
		log.Println(err)
		os.Exit(-1)
	}
	RegisterTransporterServer(grpcServer, sender)

	log.Printf("gRPC is now serving at %s\n", globalFlags.GRPCPort)
	if err := grpcServer.Serve(ln); err != nil {
		log.Println(err)
		os.Exit(-1)<commit_msg>agent: add logs for monitoring<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package agent

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/Godeps/_workspace/src/google.golang.org/grpc"
	"github.com/gyuho/psn/ps"
)

type (
	Flags struct {
		GRPCPort        string
		WorkingDir      string
		Monitor         bool
		MonitorInterval time.Duration
		MonitorFilePath string
	}

	// ZookeeperConfig is zookeeper configuration.
	// http://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
	ZookeeperConfig struct {
		TickTime       int
		DataDir        string
		ClientPort     string
		InitLimit      int
		SyncLimit      int
		PreAllocSize   int64
		MaxClientCnxns int64
		Peers          []ZookeeperPeer
	}
	ZookeeperPeer struct {
		MyID int
		IP   string
	}
)

var (
	shell = os.Getenv("SHELL")

	etcdBinaryPath = filepath.Join(os.Getenv("GOPATH"), "bin/etcd")
	etcdToken      = "etcd_token"

	dataLogPath  = "data.log"
	etcdDataDir  = "data.etcd"
	zkWorkingDir = "zookeeper"
	zkDataDir    = "zookeeper/data.zk"

	zkTemplate = `tickTime={{.TickTime}}
dataDir={{.DataDir}}
clientPort={{.ClientPort}}
initLimit={{.InitLimit}}
syncLimit={{.SyncLimit}}
preAllocSize={{.PreAllocSize}}
maxClientCnxns={{.MaxClientCnxns}}
{{range .Peers}}server.{{.MyID}}={{.IP}}:2888:3888
{{end}}
`
	zkConfig = ZookeeperConfig{
		TickTime:       2000,
		ClientPort:     "2181",
		InitLimit:      5,
		SyncLimit:      5,
		PreAllocSize:   65536 * 1024,
		MaxClientCnxns: 60,
		Peers: []ZookeeperPeer{
			{MyID: 1, IP: "10.240.0.12"},
			{MyID: 2, IP: "10.240.0.13"},
			{MyID: 3, IP: "10.240.0.14"},
		},
	}

	Command = &cobra.Command{
		Use:   "agent",
		Short: "Database agent in remote servers.",
		Run:   CommandFunc,
	}
	globalFlags = Flags{}
)

func homeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func init() {
	if len(shell) == 0 {
		shell = "sh"
	}
	Command.PersistentFlags().StringVarP(&globalFlags.GRPCPort, "agent-port", "p", ":3500", "Port to server agent gRPC server.")
	Command.PersistentFlags().StringVarP(&globalFlags.WorkingDir, "working-dir", "d", homeDir(), "Working directory to store data.")
	Command.PersistentFlags().BoolVar(&globalFlags.Monitor, "monitor", false, "Periodically records resource usage.")
	Command.PersistentFlags().DurationVar(&globalFlags.MonitorInterval, "monitor-interval", time.Second, "Resource monitor interval.")
	Command.PersistentFlags().StringVar(&globalFlags.MonitorFilePath, "monitor-path", "monitor.csv", "File path to store monitor results.")
}

func CommandFunc(cmd *cobra.Command, args []string) {
	dataLogPath = filepath.Join(globalFlags.WorkingDir, dataLogPath)
	etcdDataDir = filepath.Join(globalFlags.WorkingDir, etcdDataDir)
	zkWorkingDir = filepath.Join(globalFlags.WorkingDir, zkWorkingDir)
	zkDataDir = filepath.Join(globalFlags.WorkingDir, zkDataDir)

	log.Printf("gRPC has started serving at  %s\n", globalFlags.GRPCPort)
	log.Printf("data log path:               %s\n", dataLogPath)
	log.Printf("etcd data directory:         %s\n", etcdDataDir)
	log.Printf("Zookeeper working directory: %s\n", zkWorkingDir)
	log.Printf("Zookeeper data directory:    %s\n", zkDataDir)

	var (
		grpcServer = grpc.NewServer()
		sender     = NewTransporterServer()
	)
	ln, err := net.Listen("tcp", globalFlags.GRPCPort)
	if err != nil {
		log.Println(err)
		os.Exit(-1)
	}
	RegisterTransporterServer(grpcServer, sender)

	log.Printf("gRPC is now serving at %s\n", globalFlags.GRPCPort)
	if globalFlags.Monitor {
		log.Printf("monitor flag set 'true' -- as soon as database started, it will start monitoring every %v\n", globalFlags.MonitorInterval)
	}
	if err := grpcServer.Serve(ln); err != nil {
		log.Println(err)
		os.Exit(-1)
{'fraction_non_alphanumeric': 0.13991860081399185, 'fraction_numerical': 0.016939830601693982, 'mean_word_length': 3.8881720430107527, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3252448', 'n_tokens_mistral': 3427, 'n_tokens_neox': 3215, 'n_words': 729}