The test switched the peer's uid in a plain variable the keeper's serving goroutines read, a data race that fails it under -race; it is an atomic now. A peer the keeper refuses could also get "broken pipe" instead of the refusal, when the keeper closed before the request was written: the client now reads the refusal that waits on the socket. A request larger than the socket's buffer makes that case happen every time. Builds: A-37 (only root opens the keeper's socket; a non-root peer is told so); design 47 H section 10 rows 165 and 167 (hq branch design/47-H1-answers-59, 81fa1ee4)
208 lines
8.8 KiB
Go
208 lines
8.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
)
|
|
|
|
// A-37: the keeper answers a closed list of calls: an event of the module (never scrub.requested), a key of its three
|
|
// states, and one of three invokes (the service manager only for the module's two units); anything else is refused.
|
|
func TestTheKeeperAnswersOnlyItsClosedListOfCalls(t *testing.T) {
|
|
var emitted []string
|
|
var called, said []string
|
|
st := memStates{}
|
|
k := &Keeper{
|
|
Emit: func(kind string, body any) error { emitted = append(emitted, kind); return nil },
|
|
States: st,
|
|
Call: func(key string, body any) (json.RawMessage, error) {
|
|
called = append(called, key)
|
|
return json.RawMessage(`{"ok":true}`), nil
|
|
},
|
|
Finding: func(event string, body json.RawMessage, id string) error {
|
|
said = append(said, event+"/"+id)
|
|
return nil
|
|
},
|
|
}
|
|
req := func(op, kind, state, key, body string) KeeperRequest {
|
|
return KeeperRequest{Op: op, Kind: kind, State: state, Key: key, Body: json.RawMessage(body)}
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
r KeeperRequest
|
|
refused string
|
|
}{
|
|
{"emit a kind", req(OpEmit, EventApplied, "", "", `{"key":"x"}`), ""},
|
|
{"emit the scrub's kind", req(OpEmit, EventScrubRequested, "", "", `{}`), "not an event"},
|
|
{"emit another module's kind", req(OpEmit, "package.changed", "", "", `{}`), "not an event"},
|
|
{"put applied", req(OpPut, "", StateApplied, "hostid", `{"outcome":"applied"}`), ""},
|
|
{"put schedule", req(OpPut, "", StateSchedule, "paused", `{}`), ""},
|
|
{"put another state", req(OpPut, "", "holds", "x", `{}`), "not a state"},
|
|
{"get findings", req(OpGet, "", StateFindings, "k", ``), ""},
|
|
{"keys of another state", req(OpKeys, "", "mirrors", "", ``), "not a state"},
|
|
{"delete schedule", req(OpDelete, "", StateSchedule, "paused", ``), ""},
|
|
{"invoke record", req(OpInvoke, "", "", ToolRecord, `{}`), ""},
|
|
{"invoke schedule", req(OpInvoke, "", "", ToolSchedule, `{}`), ""},
|
|
{"start the run", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"zfs-mesh-run.service"}`), ""},
|
|
{"start the apply", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"zfs-config-apply.service"}`), ""},
|
|
{"start another unit", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"sshd.service"}`), "starts only"},
|
|
{"stop a unit", req(OpInvoke, "", "", "seat:node-service-manager.stop@ace", `{"scope":"system","unit":"zfs-mesh-run.service"}`), "not one of the three"},
|
|
{"invoke a tool of the module", req(OpInvoke, "", "", "zfs.zfs_snapshot", `{}`), "not one of the three"},
|
|
{"another op", req("shell", "", "", "", ``), "not a call"},
|
|
// A-339, A-340: the findings writer's three events, and no other, on the finding stream.
|
|
{"say a finding written", KeeperRequest{Op: OpFinding, Kind: "finding-written", Body: json.RawMessage(`{}`), ID: "w1"}, ""},
|
|
{"say the findings stated", KeeperRequest{Op: OpFinding, Kind: "findings-stated", Body: json.RawMessage(`{}`), ID: "s1"}, ""},
|
|
{"say another event as a finding", KeeperRequest{Op: OpFinding, Kind: EventApplied, Body: json.RawMessage(`{}`)}, "not an event of the findings writer"},
|
|
}
|
|
for _, c := range cases {
|
|
a := k.Do(c.r)
|
|
if (c.refused == "") != (a.Error == "") || (c.refused != "" && !strings.Contains(a.Error, c.refused)) {
|
|
t.Errorf("%s: %+v", c.name, a)
|
|
}
|
|
}
|
|
if len(said) != 2 || said[0] != "finding-written/w1" {
|
|
t.Errorf("said %v", said)
|
|
}
|
|
if len(emitted) != 1 || len(called) != 4 {
|
|
t.Errorf("emitted %v, called %v", emitted, called)
|
|
}
|
|
if _, left := st["applied/hostid"]; !left {
|
|
t.Errorf("the put was not kept")
|
|
}
|
|
}
|
|
|
|
// shortTempDir is a directory in the test's temporary directory, or in /tmp when a socket in it (`k/k.sock`) would pass 100
|
|
// characters, the most a unix socket path takes, so a long TMPDIR cannot fail the bind.
|
|
func shortTempDir(prefix string) (string, error) {
|
|
dir, err := os.MkdirTemp("", prefix)
|
|
if err != nil || len(filepath.Join(dir, "k", "k.sock")) <= 100 {
|
|
return dir, err
|
|
}
|
|
os.RemoveAll(dir)
|
|
return os.MkdirTemp("/tmp", prefix)
|
|
}
|
|
|
|
// A-37: only root opens the socket; a root process says and reads through it as through a bus of its own, and with no
|
|
// keeper answering it acts as with the bus down, with a warning (A-8).
|
|
func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing.T) {
|
|
st := memStates{}
|
|
// The peer's uid is read by the keeper's serving goroutines and switched by the test: an atomic, so no connection
|
|
// sees it half-written (issue 1960: a data race failed the test under -race).
|
|
var uid atomic.Int32
|
|
k := &Keeper{Emit: func(string, any) error { return nil }, States: st,
|
|
Call: func(string, any) (json.RawMessage, error) { return nil, errors.New("down") },
|
|
PeerOf: func(net.Conn) (int, error) { return int(uid.Load()), nil }}
|
|
dir, err := shortTempDir("zk")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
sock := filepath.Join(dir, "k", "k.sock")
|
|
l, err := Listen(sock)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, stop := context.WithCancel(context.Background())
|
|
defer stop()
|
|
go k.Serve(ctx, l)
|
|
c := keeperClient{sock: sock}
|
|
if err := c.Put(StateSchedule, "snapshots", map[string]string{"by": "x"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if keys, err := c.Keys(StateSchedule); err != nil || len(keys) != 1 || keys[0] != "snapshots" {
|
|
t.Errorf("%v %v", keys, err)
|
|
}
|
|
if v, ok, err := c.Get(StateSchedule, "snapshots"); err != nil || !ok || !strings.Contains(string(v), `"by":"x"`) {
|
|
t.Errorf("%s %v %v", v, ok, err)
|
|
}
|
|
if _, err := c.call(ToolSchedule, map[string]any{}); err == nil || !strings.Contains(err.Error(), "down") {
|
|
t.Errorf("an invoke that failed: %v", err)
|
|
}
|
|
uid.Store(1000)
|
|
if err := c.Put(StateSchedule, "x", 1); err == nil || !strings.Contains(err.Error(), "only root") {
|
|
t.Errorf("a peer that is not root: %v", err)
|
|
}
|
|
// A request larger than the socket's buffer: the keeper refuses and closes before it is written whole, so the write
|
|
// meets a closed socket every time; the peer is still told it is not root (issue 1960).
|
|
if err := c.Put(StateSchedule, "y", strings.Repeat("x", 1<<20)); err == nil || !strings.Contains(err.Error(), "only root") {
|
|
t.Errorf("a peer that is not root, its write cut: %v", err)
|
|
}
|
|
if _, left := st["schedule/x"]; left {
|
|
t.Errorf("a refused peer's call was read")
|
|
}
|
|
if _, left := st["schedule/y"]; left {
|
|
t.Errorf("a refused peer's cut call was read")
|
|
}
|
|
// No socket variable: the machine keeps no bus and says so.
|
|
t.Setenv(KeeperSocketEnv, "")
|
|
m := newFake(t).machine()
|
|
m.Emit = nil
|
|
m.JoinBus()
|
|
if m.Emit != nil || m.States != nil || len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "bus is taken as down") {
|
|
t.Errorf("no keeper: %+v", m.Warnings)
|
|
}
|
|
// A keeper not yet up: the process joins, its calls fail, and the same process uses the keeper once it is up.
|
|
late := filepath.Join(dir, "late.sock")
|
|
t.Setenv(KeeperSocketEnv, late)
|
|
ml := newFake(t).machine()
|
|
ml.JoinBus()
|
|
if ml.States == nil {
|
|
t.Fatalf("a keeper not up yet must not take the bus as down: %+v", ml.Warnings)
|
|
}
|
|
if _, err := ml.States.Keys(StateSchedule); err == nil {
|
|
t.Errorf("no keeper up, yet a call answered")
|
|
}
|
|
ll, err := Listen(late)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The socket is bound before the call below, so the call waits in its backlog until Serve accepts it: the keeper
|
|
// is up from the moment Listen returns, whenever its goroutine runs.
|
|
lctx, lcancel := context.WithCancel(context.Background())
|
|
defer lcancel()
|
|
go (&Keeper{Emit: func(string, any) error { return nil }, States: memStates{}, PeerOf: func(net.Conn) (int, error) { return 0, nil },
|
|
Call: func(string, any) (json.RawMessage, error) { return nil, nil }}).Serve(lctx, ll)
|
|
if _, err := ml.States.Keys(StateSchedule); err != nil {
|
|
t.Errorf("a keeper that came up later is not used: %v", err)
|
|
}
|
|
t.Setenv(KeeperSocketEnv, sock)
|
|
uid.Store(0)
|
|
m2 := newFake(t).machine()
|
|
m2.JoinBus()
|
|
if m2.Emit == nil || m2.States == nil || m2.Call == nil || m2.keeper == nil {
|
|
t.Errorf("a keeper answering: the bus is up")
|
|
}
|
|
}
|
|
|
|
// A-37: a root process whose keeper does not answer acts as with the bus down and says so, once, in a warning, for the
|
|
// hand acts, the forgetting of applied keys and the pause alike.
|
|
func TestAKeeperThatDoesNotAnswerIsWarnedOnce(t *testing.T) {
|
|
dir, err := shortTempDir("kd")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
t.Setenv(KeeperSocketEnv, filepath.Join(dir, "none.sock"))
|
|
m := newFake(t).machine()
|
|
m.JoinBus()
|
|
if got := m.handActs(); got != nil {
|
|
t.Errorf("hand acts with no keeper: %v", got)
|
|
}
|
|
if len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "bus is taken as down") {
|
|
t.Fatalf("hand acts: %v", m.Warnings)
|
|
}
|
|
m.forget(&Config{})
|
|
if m.Paused() {
|
|
t.Errorf("paused with no keeper")
|
|
}
|
|
if len(m.Warnings) != 1 {
|
|
t.Errorf("said more than once: %v", m.Warnings)
|
|
}
|
|
}
|