functional-tester/etcd-agent: remove

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
This commit is contained in:
Gyuho Lee
2018-03-28 18:26:14 -07:00
parent 3fba5ab403
commit f7eab57b3c
8 changed files with 0 additions and 953 deletions

View File

@@ -1,372 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"syscall"
"time"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/transport"
"github.com/coreos/etcd/tools/functional-tester/etcd-agent/client"
)
const (
stateUninitialized = "uninitialized"
stateStarted = "started"
stateStopped = "stopped"
stateTerminated = "terminated"
)
type Agent struct {
state string // the state of etcd process
cmd *exec.Cmd
logfile *os.File
cfg AgentConfig
pmu sync.Mutex
advertisePortToProxy map[int]transport.Proxy
}
type AgentConfig struct {
EtcdPath string
LogDir string
FailpointAddr string
}
func newAgent(cfg AgentConfig) (*Agent, error) {
// check if the file exists
_, err := os.Stat(cfg.EtcdPath)
if err != nil {
return nil, err
}
c := exec.Command(cfg.EtcdPath)
err = fileutil.TouchDirAll(cfg.LogDir)
if err != nil {
return nil, err
}
var f *os.File
f, err = os.Create(filepath.Join(cfg.LogDir, "etcd.log"))
if err != nil {
return nil, err
}
return &Agent{
state: stateUninitialized,
cmd: c,
logfile: f,
cfg: cfg,
advertisePortToProxy: make(map[int]transport.Proxy),
}, nil
}
// start starts a new etcd process with the given args.
func (a *Agent) start(args ...string) error {
args = append(args, "--data-dir", a.dataDir())
a.cmd = exec.Command(a.cmd.Path, args...)
a.cmd.Env = []string{"GOFAIL_HTTP=" + a.cfg.FailpointAddr}
a.cmd.Stdout = a.logfile
a.cmd.Stderr = a.logfile
err := a.cmd.Start()
if err != nil {
return err
}
a.state = stateStarted
a.pmu.Lock()
defer a.pmu.Unlock()
if len(a.advertisePortToProxy) == 0 {
// enough time for etcd start before setting up proxy
time.Sleep(time.Second)
var (
err error
s string
listenClientURL *url.URL
advertiseClientURL *url.URL
advertiseClientURLPort int
listenPeerURL *url.URL
advertisePeerURL *url.URL
advertisePeerURLPort int
)
for i := range args {
switch args[i] {
case "--listen-client-urls":
listenClientURL, err = url.Parse(args[i+1])
if err != nil {
return err
}
case "--advertise-client-urls":
advertiseClientURL, err = url.Parse(args[i+1])
if err != nil {
return err
}
_, s, err = net.SplitHostPort(advertiseClientURL.Host)
if err != nil {
return err
}
advertiseClientURLPort, err = strconv.Atoi(s)
if err != nil {
return err
}
case "--listen-peer-urls":
listenPeerURL, err = url.Parse(args[i+1])
if err != nil {
return err
}
case "--initial-advertise-peer-urls":
advertisePeerURL, err = url.Parse(args[i+1])
if err != nil {
return err
}
_, s, err = net.SplitHostPort(advertisePeerURL.Host)
if err != nil {
return err
}
advertisePeerURLPort, err = strconv.Atoi(s)
if err != nil {
return err
}
}
}
clientProxy := transport.NewProxy(transport.ProxyConfig{
From: *advertiseClientURL,
To: *listenClientURL,
})
select {
case err = <-clientProxy.Error():
return err
case <-time.After(time.Second):
}
a.advertisePortToProxy[advertiseClientURLPort] = clientProxy
peerProxy := transport.NewProxy(transport.ProxyConfig{
From: *advertisePeerURL,
To: *listenPeerURL,
})
select {
case err = <-peerProxy.Error():
return err
case <-time.After(time.Second):
}
a.advertisePortToProxy[advertisePeerURLPort] = peerProxy
}
return nil
}
// stop stops the existing etcd process the agent started.
func (a *Agent) stopWithSig(sig os.Signal) error {
if a.state != stateStarted {
return nil
}
a.pmu.Lock()
if len(a.advertisePortToProxy) > 0 {
for _, p := range a.advertisePortToProxy {
if err := p.Close(); err != nil {
a.pmu.Unlock()
return err
}
select {
case <-p.Done():
// enough time to release port
time.Sleep(time.Second)
case <-time.After(time.Second):
}
}
a.advertisePortToProxy = make(map[int]transport.Proxy)
}
a.pmu.Unlock()
err := stopWithSig(a.cmd, sig)
if err != nil {
return err
}
a.state = stateStopped
return nil
}
func stopWithSig(cmd *exec.Cmd, sig os.Signal) error {
err := cmd.Process.Signal(sig)
if err != nil {
return err
}
errc := make(chan error)
go func() {
_, ew := cmd.Process.Wait()
errc <- ew
close(errc)
}()
select {
case <-time.After(5 * time.Second):
cmd.Process.Kill()
case e := <-errc:
return e
}
err = <-errc
return err
}
// restart restarts the stopped etcd process.
func (a *Agent) restart() error {
return a.start(a.cmd.Args[1:]...)
}
func (a *Agent) cleanup() error {
// exit with stackstrace
if err := a.stopWithSig(syscall.SIGQUIT); err != nil {
return err
}
a.state = stateUninitialized
a.logfile.Close()
if err := archiveLogAndDataDir(a.cfg.LogDir, a.dataDir()); err != nil {
return err
}
if err := fileutil.TouchDirAll(a.cfg.LogDir); err != nil {
return err
}
f, err := os.Create(filepath.Join(a.cfg.LogDir, "etcd.log"))
if err != nil {
return err
}
a.logfile = f
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
// https://github.com/torvalds/linux/blob/master/fs/drop_caches.c
cmd := exec.Command("/bin/sh", "-c", `echo "echo 1 > /proc/sys/vm/drop_caches" | sudo sh`)
if err := cmd.Run(); err != nil {
plog.Infof("error when cleaning page cache (%v)", err)
}
return nil
}
// terminate stops the exiting etcd process the agent started
// and removes the data dir.
func (a *Agent) terminate() error {
err := a.stopWithSig(syscall.SIGTERM)
if err != nil {
return err
}
err = os.RemoveAll(a.dataDir())
if err != nil {
return err
}
a.state = stateTerminated
return nil
}
func (a *Agent) dropPort(port int) error {
a.pmu.Lock()
defer a.pmu.Unlock()
p, ok := a.advertisePortToProxy[port]
if !ok {
return fmt.Errorf("%d does not have proxy", port)
}
p.BlackholeTx()
p.BlackholeRx()
return nil
}
func (a *Agent) recoverPort(port int) error {
a.pmu.Lock()
defer a.pmu.Unlock()
p, ok := a.advertisePortToProxy[port]
if !ok {
return fmt.Errorf("%d does not have proxy", port)
}
p.UnblackholeTx()
p.UnblackholeRx()
return nil
}
func (a *Agent) setLatency(ms, rv int) error {
a.pmu.Lock()
defer a.pmu.Unlock()
if ms == 0 {
for _, p := range a.advertisePortToProxy {
p.UndelayTx()
p.UndelayRx()
}
}
for _, p := range a.advertisePortToProxy {
p.DelayTx(time.Duration(ms)*time.Millisecond, time.Duration(rv)*time.Millisecond)
p.DelayRx(time.Duration(ms)*time.Millisecond, time.Duration(rv)*time.Millisecond)
}
return nil
}
func (a *Agent) status() client.Status {
return client.Status{State: a.state}
}
func (a *Agent) dataDir() string {
return filepath.Join(a.cfg.LogDir, "etcd.data")
}
func existDir(fpath string) bool {
st, err := os.Stat(fpath)
if err != nil {
if os.IsNotExist(err) {
return false
}
} else {
return st.IsDir()
}
return false
}
func archiveLogAndDataDir(logDir string, datadir string) error {
dir := filepath.Join(logDir, "failure_archive", time.Now().Format(time.RFC3339))
if existDir(dir) {
dir = filepath.Join(logDir, "failure_archive", time.Now().Add(time.Second).Format(time.RFC3339))
}
if err := fileutil.TouchDirAll(dir); err != nil {
return err
}
if err := os.Rename(filepath.Join(logDir, "etcd.log"), filepath.Join(dir, "etcd.log")); err != nil {
if !os.IsNotExist(err) {
return err
}
}
if err := os.Rename(datadir, filepath.Join(dir, filepath.Base(datadir))); err != nil {
if !os.IsNotExist(err) {
return err
}
}
return nil
}

View File

@@ -1,87 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"os"
"path/filepath"
"syscall"
"testing"
)
var etcdPath = filepath.Join(os.Getenv("GOPATH"), "bin/etcd")
func TestAgentStart(t *testing.T) {
defer os.Remove("etcd.log")
a := newTestAgent(t)
defer a.terminate()
err := a.start()
if err != nil {
t.Fatal(err)
}
}
func TestAgentRestart(t *testing.T) {
defer os.Remove("etcd.log")
a := newTestAgent(t)
defer a.terminate()
err := a.start()
if err != nil {
t.Fatal(err)
}
err = a.stopWithSig(syscall.SIGTERM)
if err != nil {
t.Fatal(err)
}
err = a.restart()
if err != nil {
t.Fatal(err)
}
}
func TestAgentTerminate(t *testing.T) {
defer os.Remove("etcd.log")
a := newTestAgent(t)
err := a.start()
if err != nil {
t.Fatal(err)
}
err = a.terminate()
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(a.dataDir()); !os.IsNotExist(err) {
t.Fatal(err)
}
}
// newTestAgent creates a test agent
func newTestAgent(t *testing.T) *Agent {
a, err := newAgent(AgentConfig{EtcdPath: etcdPath, LogDir: "etcd.log"})
if err != nil {
t.Fatal(err)
}
return a
}

View File

@@ -1,118 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package client
import "net/rpc"
type Status struct {
// State gives the human-readable status of an agent (e.g., "started" or "terminated")
State string
// TODO: gather more informations
// TODO: memory usage, raft information, etc..
}
type Agent interface {
ID() uint64
// Start starts a new etcd with the given args on the agent machine.
Start(args ...string) (int, error)
// Stop stops the existing etcd the agent started.
Stop() error
// Restart restarts the existing etcd the agent stopped.
Restart() (int, error)
// Cleanup stops the exiting etcd the agent started, then archives log and its data dir.
Cleanup() error
// Terminate stops the exiting etcd the agent started and removes its data dir.
Terminate() error
// DropPort drops all network packets at the given port.
DropPort(port int) error
// RecoverPort stops dropping all network packets at the given port.
RecoverPort(port int) error
// SetLatency slows down network by introducing latency.
SetLatency(ms, rv int) error
// RemoveLatency removes latency introduced by SetLatency.
RemoveLatency() error
// Status returns the status of etcd on the agent
Status() (Status, error)
}
type agent struct {
endpoint string
rpcClient *rpc.Client
}
func NewAgent(endpoint string) (Agent, error) {
c, err := rpc.DialHTTP("tcp", endpoint)
if err != nil {
return nil, err
}
return &agent{endpoint, c}, nil
}
func (a *agent) Start(args ...string) (int, error) {
var pid int
err := a.rpcClient.Call("Agent.RPCStart", args, &pid)
if err != nil {
return -1, err
}
return pid, nil
}
func (a *agent) Stop() error {
return a.rpcClient.Call("Agent.RPCStop", struct{}{}, nil)
}
func (a *agent) Restart() (int, error) {
var pid int
err := a.rpcClient.Call("Agent.RPCRestart", struct{}{}, &pid)
if err != nil {
return -1, err
}
return pid, nil
}
func (a *agent) Cleanup() error {
return a.rpcClient.Call("Agent.RPCCleanup", struct{}{}, nil)
}
func (a *agent) Terminate() error {
return a.rpcClient.Call("Agent.RPCTerminate", struct{}{}, nil)
}
func (a *agent) DropPort(port int) error {
return a.rpcClient.Call("Agent.RPCDropPort", port, nil)
}
func (a *agent) RecoverPort(port int) error {
return a.rpcClient.Call("Agent.RPCRecoverPort", port, nil)
}
func (a *agent) SetLatency(ms, rv int) error {
return a.rpcClient.Call("Agent.RPCSetLatency", []int{ms, rv}, nil)
}
func (a *agent) RemoveLatency() error {
return a.rpcClient.Call("Agent.RPCRemoveLatency", struct{}{}, nil)
}
func (a *agent) Status() (Status, error) {
var s Status
err := a.rpcClient.Call("Agent.RPCStatus", struct{}{}, &s)
return s, err
}
func (a *agent) ID() uint64 {
panic("not implemented")
}

View File

@@ -1,16 +0,0 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package client provides a client implementation to control an etcd-agent.
package client

View File

@@ -1,16 +0,0 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// etcd-agent is a daemon for controlling an etcd process via HTTP RPC.
package main

View File

@@ -1,47 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"os"
"path/filepath"
"github.com/coreos/pkg/capnslog"
)
var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "etcd-agent")
func main() {
etcdPath := flag.String("etcd-path", filepath.Join(os.Getenv("GOPATH"), "bin/etcd"), "the path to etcd binary")
etcdLogDir := flag.String("etcd-log-dir", "etcd-log", "directory to store etcd logs, data directories, failure archive")
port := flag.String("port", ":9027", "port to serve agent server")
failpointAddr := flag.String("failpoint-addr", ":2381", "interface for gofail's HTTP server")
flag.Parse()
cfg := AgentConfig{
EtcdPath: *etcdPath,
LogDir: *etcdLogDir,
FailpointAddr: *failpointAddr,
}
a, err := newAgent(cfg)
if err != nil {
plog.Fatal(err)
}
a.serveRPC(*port)
var done chan struct{}
<-done
}

View File

@@ -1,131 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"net"
"net/http"
"net/rpc"
"syscall"
"github.com/coreos/etcd/tools/functional-tester/etcd-agent/client"
)
func (a *Agent) serveRPC(port string) {
rpc.Register(a)
rpc.HandleHTTP()
l, e := net.Listen("tcp", port)
if e != nil {
plog.Fatal(e)
}
plog.Println("agent listening on", port)
go http.Serve(l, nil)
}
func (a *Agent) RPCStart(args []string, pid *int) error {
plog.Printf("start etcd with args %v", args)
err := a.start(args...)
if err != nil {
plog.Println("error starting etcd", err)
return err
}
*pid = a.cmd.Process.Pid
return nil
}
func (a *Agent) RPCStop(args struct{}, reply *struct{}) error {
plog.Printf("stop etcd")
err := a.stopWithSig(syscall.SIGTERM)
if err != nil {
plog.Println("error stopping etcd", err)
return err
}
return nil
}
func (a *Agent) RPCRestart(args struct{}, pid *int) error {
plog.Printf("restart etcd")
err := a.restart()
if err != nil {
plog.Println("error restarting etcd", err)
return err
}
*pid = a.cmd.Process.Pid
return nil
}
func (a *Agent) RPCCleanup(args struct{}, reply *struct{}) error {
plog.Printf("cleanup etcd")
err := a.cleanup()
if err != nil {
plog.Println("error cleaning up etcd", err)
return err
}
return nil
}
func (a *Agent) RPCTerminate(args struct{}, reply *struct{}) error {
plog.Printf("terminate etcd")
err := a.terminate()
if err != nil {
plog.Println("error terminating etcd", err)
}
return nil
}
func (a *Agent) RPCDropPort(port int, reply *struct{}) error {
plog.Printf("drop port %d", port)
err := a.dropPort(port)
if err != nil {
plog.Println("error dropping port", err)
}
return nil
}
func (a *Agent) RPCRecoverPort(port int, reply *struct{}) error {
plog.Printf("recover port %d", port)
err := a.recoverPort(port)
if err != nil {
plog.Println("error recovering port", err)
}
return nil
}
func (a *Agent) RPCSetLatency(args []int, reply *struct{}) error {
if len(args) != 2 {
return fmt.Errorf("SetLatency needs two args, got (%v)", args)
}
plog.Printf("set latency of %dms (+/- %dms)", args[0], args[1])
err := a.setLatency(args[0], args[1])
if err != nil {
plog.Println("error setting latency", err)
}
return nil
}
func (a *Agent) RPCRemoveLatency(args struct{}, reply *struct{}) error {
plog.Println("removing latency")
err := a.setLatency(0, 0)
if err != nil {
plog.Println("error removing latency")
}
return nil
}
func (a *Agent) RPCStatus(args struct{}, status *client.Status) error {
*status = a.status()
return nil
}

View File

@@ -1,166 +0,0 @@
// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"io/ioutil"
"log"
"net/rpc"
"os"
"testing"
"github.com/coreos/etcd/tools/functional-tester/etcd-agent/client"
)
func init() {
defaultAgent, err := newAgent(AgentConfig{EtcdPath: etcdPath, LogDir: "etcd.log"})
if err != nil {
log.Panic(err)
}
defaultAgent.serveRPC(":9027")
}
func TestRPCStart(t *testing.T) {
c, err := rpc.DialHTTP("tcp", ":9027")
if err != nil {
t.Fatal(err)
}
dir, err := ioutil.TempDir(os.TempDir(), "etcd-agent")
if err != nil {
t.Fatal(err)
}
var pid int
err = c.Call("Agent.RPCStart", []string{"--data-dir", dir}, &pid)
if err != nil {
t.Fatal(err)
}
defer c.Call("Agent.RPCTerminate", struct{}{}, nil)
_, err = os.FindProcess(pid)
if err != nil {
t.Errorf("unexpected error %v when find process %d", err, pid)
}
}
func TestRPCRestart(t *testing.T) {
c, err := rpc.DialHTTP("tcp", ":9027")
if err != nil {
t.Fatal(err)
}
dir, err := ioutil.TempDir(os.TempDir(), "etcd-agent")
if err != nil {
t.Fatal(err)
}
var pid int
err = c.Call("Agent.RPCStart", []string{"--data-dir", dir}, &pid)
if err != nil {
t.Fatal(err)
}
defer c.Call("Agent.RPCTerminate", struct{}{}, nil)
err = c.Call("Agent.RPCStop", struct{}{}, nil)
if err != nil {
t.Fatal(err)
}
var npid int
err = c.Call("Agent.RPCRestart", struct{}{}, &npid)
if err != nil {
t.Fatal(err)
}
if npid == pid {
t.Errorf("pid = %v, want not equal to %d", npid, pid)
}
s, err := os.FindProcess(pid)
if err != nil {
t.Errorf("unexpected error %v when find process %d", err, pid)
}
_, err = s.Wait()
if err == nil {
t.Errorf("err = nil, want killed error")
}
_, err = os.FindProcess(npid)
if err != nil {
t.Errorf("unexpected error %v when find process %d", err, npid)
}
}
func TestRPCTerminate(t *testing.T) {
c, err := rpc.DialHTTP("tcp", ":9027")
if err != nil {
t.Fatal(err)
}
dir, err := ioutil.TempDir(os.TempDir(), "etcd-agent")
if err != nil {
t.Fatal(err)
}
var pid int
err = c.Call("Agent.RPCStart", []string{"--data-dir", dir}, &pid)
if err != nil {
t.Fatal(err)
}
err = c.Call("Agent.RPCTerminate", struct{}{}, nil)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatal(err)
}
}
func TestRPCStatus(t *testing.T) {
c, err := rpc.DialHTTP("tcp", ":9027")
if err != nil {
t.Fatal(err)
}
var s client.Status
err = c.Call("Agent.RPCStatus", struct{}{}, &s)
if err != nil {
t.Fatal(err)
}
if s.State != stateTerminated {
t.Errorf("state = %s, want %s", s.State, stateTerminated)
}
dir, err := ioutil.TempDir(os.TempDir(), "etcd-agent")
if err != nil {
t.Fatal(err)
}
var pid int
err = c.Call("Agent.RPCStart", []string{"--data-dir", dir}, &pid)
if err != nil {
t.Fatal(err)
}
err = c.Call("Agent.RPCStatus", struct{}{}, &s)
if err != nil {
t.Fatal(err)
}
if s.State != stateStarted {
t.Errorf("state = %s, want %s", s.State, stateStarted)
}
err = c.Call("Agent.RPCTerminate", struct{}{}, nil)
if err != nil {
t.Fatal(err)
}
}