<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package agent

import (
	"bytes"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/coreos/dbtester/remotestorage"
	"github.com/coreos/pkg/capnslog"
	"github.com/gyuho/psn/process"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type Flags struct {
	GRPCPort         string
	WorkingDirectory string
}

// ZookeeperConfig is zookeeper configuration.
// http://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
type ZookeeperConfig struct {
	TickTime       int
	DataDir        string
	ClientPort     string
	InitLimit      int
	SyncLimit      int
	MaxClientCnxns int64
	SnapCount      int64
	Peers          []ZookeeperPeer
}

// ZookeeperPeer defines Zookeeper peer configuration.
type ZookeeperPeer struct {
	MyID int
	IP   string
}

var (
	shell = os.Getenv("SHELL")

	agentLogPath    = "agent.log"
	databaseLogPath = "database.log"
	monitorLogPath  = "monitor.csv"

	etcdBinaryPath   = filepath.Join(os.Getenv("GOPATH"), "bin/etcd")
	consulBinaryPath = filepath.Join(os.Getenv("GOPATH"), "bin/consul")
	zetcdBinaryPath  = filepath.Join(os.Getenv("GOPATH"), "bin/zetcd")
	cetcdBinaryPath  = filepath.Join(os.Getenv("GOPATH"), "bin/cetcd")

	javaBinaryPath = "/usr/bin/java"

	etcdToken     = "etcd_token"
	etcdDataDir   = "data.etcd"
	consulDataDir = "data.consul"

	zkWorkingDir = "zookeeper"
	zkDataDir    = "zookeeper/data.zk"
	zkConfigPath = "zookeeper.config"
	zkTemplate   = `tickTime={{.TickTime}}
dataDir={{.DataDir}}
clientPort={{.ClientPort}}
initLimit={{.InitLimit}}
syncLimit={{.SyncLimit}}
maxClientCnxns={{.MaxClientCnxns}}
snapCount={{.SnapCount}}
{{range .Peers}}server.{{.MyID}}={{.IP}}:2888:3888
{{end}}
`
	zkConfigDefault = ZookeeperConfig{
		TickTime:       2000,
		ClientPort:     "2181",
		InitLimit:      5,
		SyncLimit:      5,
		MaxClientCnxns: 60,
		Peers: []ZookeeperPeer{
			{MyID: 1, IP: ""},
			{MyID: 2, IP: ""},
			{MyID: 3, IP: ""},
		},
	}
)

var (
	Command = &cobra.Command{
		Use:   "agent",
		Short: "Database agent in remote servers.",
		RunE:  CommandFunc,
	}

	globalFlags = Flags{}
)

func init() {
	if len(shell) == 0 {
		shell = "sh"
	}
	Command.PersistentFlags().StringVar(&globalFlags.GRPCPort, "agent-port", ":3500", "Port to server agent gRPC server.")
	Command.PersistentFlags().StringVar(&globalFlags.WorkingDirectory, "working-directory", homeDir(), "Working directory.")
}

func CommandFunc(cmd *cobra.Command, args []string) error {
	if !exist(globalFlags.WorkingDirectory) {
		return fmt.Errorf("%s does not exist", globalFlags.WorkingDirectory)
	}
	if !filepath.HasPrefix(agentLogPath, globalFlags.WorkingDirectory) {
		agentLogPath = filepath.Join(globalFlags.WorkingDirectory, agentLogPath)
	}

	f, err := openToAppend(agentLogPath)
	if err != nil {
		return err
	}
	defer f.Close()

	capnslog.SetFormatter(capnslog.NewPrettyFormatter(f, false))

	plog.Infof("started serving gRPC %s", globalFlags.GRPCPort)

	var (
		grpcServer = grpc.NewServer()
		sender     = NewTransporterServer()
	)
	ln, err := net.Listen("tcp", globalFlags.GRPCPort)
	if err != nil {
		return err
	}

	RegisterTransporterServer(grpcServer, sender)

	return grpcServer.Serve(ln)
}

type transporterServer struct { // satisfy TransporterServer
	req Request

	cmd     *exec.Cmd
	logfile *os.File
	pid     int

	proxyCmd     *exec.Cmd
	proxyLogfile *os.File
	proxyPid     int
}

var uploadSig = make(chan Request_Operation, 1)

func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response, error) {
	peerIPs := strings.Split(r.PeerIPString, "___")
	if r.Operation == Request_Start {
		if !filepath.HasPrefix(etcdDataDir, globalFlags.WorkingDirectory) {
			etcdDataDir = filepath.Join(globalFlags.WorkingDirectory, etcdDataDir)
		}
		if !filepath.HasPrefix(consulDataDir, globalFlags.WorkingDirectory) {
			consulDataDir = filepath.Join(globalFlags.WorkingDirectory, consulDataDir)
		}
		if !filepath.HasPrefix(zkWorkingDir, globalFlags.WorkingDirectory) {
			zkWorkingDir = filepath.Join(globalFlags.WorkingDirectory, zkWorkingDir)
		}
		if !filepath.HasPrefix(zkDataDir, globalFlags.WorkingDirectory) {
			zkDataDir = filepath.Join(globalFlags.WorkingDirectory, zkDataDir)
		}
		if !filepath.HasPrefix(databaseLogPath, globalFlags.WorkingDirectory) {
			databaseLogPath = filepath.Join(globalFlags.WorkingDirectory, databaseLogPath)
		}
		if !filepath.HasPrefix(monitorLogPath, globalFlags.WorkingDirectory) {
			monitorLogPath = filepath.Join(globalFlags.WorkingDirectory, monitorLogPath)
		}

		plog.Info("received gRPC request")
		plog.Infof("working_directory: %q", globalFlags.WorkingDirectory)
		plog.Infof("working_directory_zookeeper: %q", zkWorkingDir)
		plog.Infof("data_directory_etcd: %q", etcdDataDir)
		plog.Infof("data_directory_consul: %q", consulDataDir)
		plog.Infof("data_directory_zookeeper: %q", zkDataDir)
		plog.Infof("database_log_path: %q", databaseLogPath)
		plog.Infof("monitor_log_path: %q", monitorLogPath)
	}

	switch r.Operation {
	case Request_Start:
		t.req = *r
	case Request_UploadLog:
		t.req.GoogleCloudProjectName = r.GoogleCloudProjectName
		t.req.GoogleCloudStorageBucketName = r.GoogleCloudStorageBucketName
		t.req.GoogleCloudStorageSubDirectory = r.GoogleCloudStorageSubDirectory
	}

	if t.req.GoogleCloudStorageKey != "" {
		if err := toFile(t.req.GoogleCloudStorageKey, filepath.Join(globalFlags.WorkingDirectory, "gcloud-key.json")); err != nil {
			return nil, err
		}
	}

	var pidToMonitor int
	switch r.Operation {
	case Request_Start:
		switch t.req.Database {
		case Request_etcdv2, Request_etcdv3, Request_zetcd, Request_cetcd:
			if !exist(etcdBinaryPath) {
				return nil, fmt.Errorf("etcd binary %q does not exist", etcdBinaryPath)
			}
			if t.req.Database == Request_zetcd && !exist(zetcdBinaryPath) {
				return nil, fmt.Errorf("zetcd binary %q does not exist", zetcdBinaryPath)
			}
			if t.req.Database == Request_cetcd && !exist(cetcdBinaryPath) {
				return nil, fmt.Errorf("cetcd binary %q does not exist", cetcdBinaryPath)
			}
			if err := os.RemoveAll(etcdDataDir); err != nil {
				return nil, err
			}
			f, err := openToAppend(databaseLogPath)
			if err != nil {
				return nil, err
			}
			t.logfile = f

			clusterN := len(peerIPs)
			names := make([]string, clusterN)
			clientURLs := make([]string, clusterN)
			peerURLs := make([]string, clusterN)
			members := make([]string, clusterN)
			for i, u := range peerIPs {
				names[i] = fmt.Sprintf("etcd-%d", i+1)
				clientURLs[i] = fmt.Sprintf("http://%s:2379", u)
				peerURLs[i] = fmt.Sprintf("http://%s:2380", u)
				members[i] = fmt.Sprintf("%s=%s", names[i], peerURLs[i])
			}
			clusterStr := strings.Join(members, ",")
			flags := []string{
				"--name", names[t.req.ServerIndex],
				"--data-dir", etcdDataDir,

				"--listen-client-urls", clientURLs[t.req.ServerIndex],
				"--advertise-client-urls", clientURLs[t.req.ServerIndex],

				"--listen-peer-urls", peerURLs[t.req.ServerIndex],
				"--initial-advertise-peer-urls", peerURLs[t.req.ServerIndex],

				"--initial-cluster-token", etcdToken,
				"--initial-cluster", clusterStr,
				"--initial-cluster-state", "new",
			}
			flagString := strings.Join(flags, " ")

			cmd := exec.Command(etcdBinaryPath, flags...)
			cmd.Stdout = f
			cmd.Stderr = f

			cmdString := fmt.Sprintf("%s %s", cmd.Path, flagString)
			plog.Infof("starting binary %q", cmdString)
			if err := cmd.Start(); err != nil {
				return nil, err
			}
			t.cmd = cmd
			t.pid = cmd.Process.Pid
			plog.Infof("started binary %q [PID: %d]", cmdString, t.pid)
			pidToMonitor = t.pid
			go func() {
				if err := cmd.Wait(); err != nil {
					plog.Errorf("cmd.Wait %q returned error %v", cmdString, err)
					return
				}
				plog.Infof("exiting %q", cmdString)
			}()

			if t.req.Database == Request_zetcd || t.req.Database == Request_cetcd {
				f2, err := openToAppend(databaseLogPath + "-" + t.req.Database.String())
				if err != nil {
					return nil, err
				}
				t.proxyLogfile = f2
				var flags2 []string
				if t.req.Database == Request_zetcd {
					flags2 = []string{
						"-zkaddr", "0.0.0.0:2181",
						"-endpoint", clientURLs[t.req.ServerIndex], // etcd endpoint
					}
				} else {
					flags2 = []string{
						"-consuladdr", "0.0.0.0:8500",
						"-etcd", clientURLs[t.req.ServerIndex], // etcd endpoint
					}
				}
				flagString2 := strings.Join(flags2, " ")

				bpath := zetcdBinaryPath
				if t.req.Database == Request_cetcd {
					bpath = cetcdBinaryPath
				}
				cmd2 := exec.Command(bpath, flags2...)
				cmd2.Stdout = f2
				cmd2.Stderr = f2

				cmdString2 := fmt.Sprintf("%s %s", cmd2.Path, flagString2)
				plog.Infof("starting binary %q", cmdString2)
				if err := cmd2.Start(); err != nil {
					return nil, err
				}
				t.proxyCmd = cmd2
				t.proxyPid = cmd2.Process.Pid
				plog.Infof("started binary %q [PID: %d]", cmdString2, t.proxyPid)
				go func() {
					if err := cmd2.Wait(); err != nil {
						plog.Errorf("cmd.Wait %q returned error %v", cmdString2, err)
						return
					}
					plog.Infof("exiting %q", cmdString2)
				}()
			}

		case Request_ZooKeeper:
			if !exist(javaBinaryPath) {<commit_msg>agent: use localhost for cetcd, zetcd<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package agent

import (
	"bytes"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/coreos/dbtester/remotestorage"
	"github.com/coreos/pkg/capnslog"
	"github.com/gyuho/psn/process"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type Flags struct {
	GRPCPort         string
	WorkingDirectory string
}

// ZookeeperConfig is zookeeper configuration.
// http://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
type ZookeeperConfig struct {
	TickTime       int
	DataDir        string
	ClientPort     string
	InitLimit      int
	SyncLimit      int
	MaxClientCnxns int64
	SnapCount      int64
	Peers          []ZookeeperPeer
}

// ZookeeperPeer defines Zookeeper peer configuration.
type ZookeeperPeer struct {
	MyID int
	IP   string
}

var (
	shell = os.Getenv("SHELL")

	agentLogPath    = "agent.log"
	databaseLogPath = "database.log"
	monitorLogPath  = "monitor.csv"

	etcdBinaryPath   = filepath.Join(os.Getenv("GOPATH"), "bin/etcd")
	consulBinaryPath = filepath.Join(os.Getenv("GOPATH"), "bin/consul")
	zetcdBinaryPath  = filepath.Join(os.Getenv("GOPATH"), "bin/zetcd")
	cetcdBinaryPath  = filepath.Join(os.Getenv("GOPATH"), "bin/cetcd")

	javaBinaryPath = "/usr/bin/java"

	etcdToken     = "etcd_token"
	etcdDataDir   = "data.etcd"
	consulDataDir = "data.consul"

	zkWorkingDir = "zookeeper"
	zkDataDir    = "zookeeper/data.zk"
	zkConfigPath = "zookeeper.config"
	zkTemplate   = `tickTime={{.TickTime}}
dataDir={{.DataDir}}
clientPort={{.ClientPort}}
initLimit={{.InitLimit}}
syncLimit={{.SyncLimit}}
maxClientCnxns={{.MaxClientCnxns}}
snapCount={{.SnapCount}}
{{range .Peers}}server.{{.MyID}}={{.IP}}:2888:3888
{{end}}
`
	zkConfigDefault = ZookeeperConfig{
		TickTime:       2000,
		ClientPort:     "2181",
		InitLimit:      5,
		SyncLimit:      5,
		MaxClientCnxns: 60,
		Peers: []ZookeeperPeer{
			{MyID: 1, IP: ""},
			{MyID: 2, IP: ""},
			{MyID: 3, IP: ""},
		},
	}
)

var (
	Command = &cobra.Command{
		Use:   "agent",
		Short: "Database agent in remote servers.",
		RunE:  CommandFunc,
	}

	globalFlags = Flags{}
)

func init() {
	if len(shell) == 0 {
		shell = "sh"
	}
	Command.PersistentFlags().StringVar(&globalFlags.GRPCPort, "agent-port", ":3500", "Port to server agent gRPC server.")
	Command.PersistentFlags().StringVar(&globalFlags.WorkingDirectory, "working-directory", homeDir(), "Working directory.")
}

func CommandFunc(cmd *cobra.Command, args []string) error {
	if !exist(globalFlags.WorkingDirectory) {
		return fmt.Errorf("%s does not exist", globalFlags.WorkingDirectory)
	}
	if !filepath.HasPrefix(agentLogPath, globalFlags.WorkingDirectory) {
		agentLogPath = filepath.Join(globalFlags.WorkingDirectory, agentLogPath)
	}

	f, err := openToAppend(agentLogPath)
	if err != nil {
		return err
	}
	defer f.Close()

	capnslog.SetFormatter(capnslog.NewPrettyFormatter(f, false))

	plog.Infof("started serving gRPC %s", globalFlags.GRPCPort)

	var (
		grpcServer = grpc.NewServer()
		sender     = NewTransporterServer()
	)
	ln, err := net.Listen("tcp", globalFlags.GRPCPort)
	if err != nil {
		return err
	}

	RegisterTransporterServer(grpcServer, sender)

	return grpcServer.Serve(ln)
}

type transporterServer struct { // satisfy TransporterServer
	req Request

	cmd     *exec.Cmd
	logfile *os.File
	pid     int

	proxyCmd     *exec.Cmd
	proxyLogfile *os.File
	proxyPid     int
}

var uploadSig = make(chan Request_Operation, 1)

func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response, error) {
	peerIPs := strings.Split(r.PeerIPString, "___")
	if r.Operation == Request_Start {
		if !filepath.HasPrefix(etcdDataDir, globalFlags.WorkingDirectory) {
			etcdDataDir = filepath.Join(globalFlags.WorkingDirectory, etcdDataDir)
		}
		if !filepath.HasPrefix(consulDataDir, globalFlags.WorkingDirectory) {
			consulDataDir = filepath.Join(globalFlags.WorkingDirectory, consulDataDir)
		}
		if !filepath.HasPrefix(zkWorkingDir, globalFlags.WorkingDirectory) {
			zkWorkingDir = filepath.Join(globalFlags.WorkingDirectory, zkWorkingDir)
		}
		if !filepath.HasPrefix(zkDataDir, globalFlags.WorkingDirectory) {
			zkDataDir = filepath.Join(globalFlags.WorkingDirectory, zkDataDir)
		}
		if !filepath.HasPrefix(databaseLogPath, globalFlags.WorkingDirectory) {
			databaseLogPath = filepath.Join(globalFlags.WorkingDirectory, databaseLogPath)
		}
		if !filepath.HasPrefix(monitorLogPath, globalFlags.WorkingDirectory) {
			monitorLogPath = filepath.Join(globalFlags.WorkingDirectory, monitorLogPath)
		}

		plog.Info("received gRPC request")
		plog.Infof("working_directory: %q", globalFlags.WorkingDirectory)
		plog.Infof("working_directory_zookeeper: %q", zkWorkingDir)
		plog.Infof("data_directory_etcd: %q", etcdDataDir)
		plog.Infof("data_directory_consul: %q", consulDataDir)
		plog.Infof("data_directory_zookeeper: %q", zkDataDir)
		plog.Infof("database_log_path: %q", databaseLogPath)
		plog.Infof("monitor_log_path: %q", monitorLogPath)
	}

	switch r.Operation {
	case Request_Start:
		t.req = *r
	case Request_UploadLog:
		t.req.GoogleCloudProjectName = r.GoogleCloudProjectName
		t.req.GoogleCloudStorageBucketName = r.GoogleCloudStorageBucketName
		t.req.GoogleCloudStorageSubDirectory = r.GoogleCloudStorageSubDirectory
	}

	if t.req.GoogleCloudStorageKey != "" {
		if err := toFile(t.req.GoogleCloudStorageKey, filepath.Join(globalFlags.WorkingDirectory, "gcloud-key.json")); err != nil {
			return nil, err
		}
	}

	var pidToMonitor int
	switch r.Operation {
	case Request_Start:
		switch t.req.Database {
		case Request_etcdv2, Request_etcdv3, Request_zetcd, Request_cetcd:
			if !exist(etcdBinaryPath) {
				return nil, fmt.Errorf("etcd binary %q does not exist", etcdBinaryPath)
			}
			if t.req.Database == Request_zetcd && !exist(zetcdBinaryPath) {
				return nil, fmt.Errorf("zetcd binary %q does not exist", zetcdBinaryPath)
			}
			if t.req.Database == Request_cetcd && !exist(cetcdBinaryPath) {
				return nil, fmt.Errorf("cetcd binary %q does not exist", cetcdBinaryPath)
			}
			if err := os.RemoveAll(etcdDataDir); err != nil {
				return nil, err
			}
			f, err := openToAppend(databaseLogPath)
			if err != nil {
				return nil, err
			}
			t.logfile = f

			clusterN := len(peerIPs)
			names := make([]string, clusterN)
			clientURLs := make([]string, clusterN)
			peerURLs := make([]string, clusterN)
			members := make([]string, clusterN)
			for i, u := range peerIPs {
				names[i] = fmt.Sprintf("etcd-%d", i+1)
				clientURLs[i] = fmt.Sprintf("http://%s:2379", u)
				peerURLs[i] = fmt.Sprintf("http://%s:2380", u)
				members[i] = fmt.Sprintf("%s=%s", names[i], peerURLs[i])
			}
			clusterStr := strings.Join(members, ",")
			flags := []string{
				"--name", names[t.req.ServerIndex],
				"--data-dir", etcdDataDir,

				"--listen-client-urls", clientURLs[t.req.ServerIndex],
				"--advertise-client-urls", clientURLs[t.req.ServerIndex],

				"--listen-peer-urls", peerURLs[t.req.ServerIndex],
				"--initial-advertise-peer-urls", peerURLs[t.req.ServerIndex],

				"--initial-cluster-token", etcdToken,
				"--initial-cluster", clusterStr,
				"--initial-cluster-state", "new",
			}
			flagString := strings.Join(flags, " ")

			cmd := exec.Command(etcdBinaryPath, flags...)
			cmd.Stdout = f
			cmd.Stderr = f

			cmdString := fmt.Sprintf("%s %s", cmd.Path, flagString)
			plog.Infof("starting binary %q", cmdString)
			if err := cmd.Start(); err != nil {
				return nil, err
			}
			t.cmd = cmd
			t.pid = cmd.Process.Pid
			plog.Infof("started binary %q [PID: %d]", cmdString, t.pid)
			pidToMonitor = t.pid
			go func() {
				if err := cmd.Wait(); err != nil {
					plog.Errorf("cmd.Wait %q returned error %v", cmdString, err)
					return
				}
				plog.Infof("exiting %q", cmdString)
			}()

			if t.req.Database == Request_zetcd || t.req.Database == Request_cetcd {
				f2, err := openToAppend(databaseLogPath + "-" + t.req.Database.String())
				if err != nil {
					return nil, err
				}
				t.proxyLogfile = f2
				var flags2 []string
				if t.req.Database == Request_zetcd {
					flags2 = []string{
						// "-zkaddr", "0.0.0.0:2181",
						"-zkaddr", "localhost:2181",
						// "-endpoint", clientURLs[t.req.ServerIndex],
						"-endpoint", "localhost:2379", // etcd endpoint
					}
				} else {
					flags2 = []string{
						// "-consuladdr", "0.0.0.0:8500",
						"-consuladdr", "localhost:8500",
						// "-etcd", clientURLs[t.req.ServerIndex],
						"-etcd", "localhost:2379", // etcd endpoint
					}
				}
				flagString2 := strings.Join(flags2, " ")

				bpath := zetcdBinaryPath
				if t.req.Database == Request_cetcd {
					bpath = cetcdBinaryPath
				}
				cmd2 := exec.Command(bpath, flags2...)
				cmd2.Stdout = f2
				cmd2.Stderr = f2

				cmdString2 := fmt.Sprintf("%s %s", cmd2.Path, flagString2)
				plog.Infof("starting proxy binary %q", cmdString2)
				if err := cmd2.Start(); err != nil {
					return nil, err
				}
				t.proxyCmd = cmd2
				t.proxyPid = cmd2.Process.Pid
				plog.Infof("started proxy binary %q [PID: %d]", cmdString2, t.proxyPid)
				go func() {
					if err := cmd2.Wait(); err != nil {
						plog.Errorf("cmd.Wait %q returned error %v", cmdString2, err)
						return
					}
					plog.Infof("exiting %q", cmdString2)
				}()
			}

		case Request_ZooKeeper:
			if !exist(javaBinaryPath) {
{'fraction_non_alphanumeric': 0.1405822091490009, 'fraction_numerical': 0.010345876863779288, 'mean_word_length': 3.7333173307729237, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5235892', 'n_tokens_mistral': 7902, 'n_tokens_neox': 6922, 'n_words': 1449}