Files
mesh-delivery/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go
T
jschoubben 09e736a687 B1 (view): the view's module is its credential's, in the process and in the tools
viewstate.ViewModule is gone; the process takes cred.Module for its reader and emitter, the tools the served-as module.

Builds: design 47 G A-296, §428

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 23:38:40 +01:00

1021 lines
42 KiB
Go

//go:build integration
package main
import (
"context"
"encoding/json"
"io"
"maps"
"mesh-delivery-view/internal/view"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// serveOwners serves the six states of the indexer and the assigner the view's process follows by their first views
// (ADR 0615 §1, DLV-M4): each owner's read tool and views family, with the view's durable consumer (`n_mesh-delivery-view`).
// A bucket the test made is kept; the others are made empty.
// testView is the module the bed's credential names; the view names its buckets and events from it (A-296).
const testView = "mesh-delivery-view"
func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) {
t.Helper()
for _, o := range ownersFollowed {
viewstest.Serve(t, ctx, js, nc, o.owner, o.state, o.tool, "n_"+testView)
}
// Design 47 G §717, A-303: the seats' views at their holders, which the seat-holders view of the catalog half's claimant
// names (serveCatalog's claimant): stand-ins named for their seats, so each real bucket is the picture's seat-form name.
for _, o := range []struct{ holder, state string }{{claimantModule, view.AssignmentsState}, {"mesh-wants", "wants"}, {"mesh-wants", "asks"}} {
viewstest.Serve(t, ctx, js, nc, o.holder, o.state, o.holder+"."+o.state, "n_"+testView)
}
holders := viewstest.Serve(t, ctx, js, nc, claimantModule, "seat-holders", claimantModule+".seat-holders", "n_"+testView)
for seat, holder := range map[string]string{"mesh-module-assigner": claimantModule, "mesh-wants": "mesh-wants"} {
if _, err := holders.Put(ctx, seat, []byte(`{"seat":"`+seat+`","holders":[{"module":"`+holder+`","node":"n"}]}`)); err != nil {
t.Fatal(err)
}
}
}
// claimantModule is the stand-in module claiming mesh-module-assigner in the catalog half the view follows (A-298).
const claimantModule = "mesh-module-assigner"
// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the
// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the
// view's durable consumer hears.
type catalogHolder struct {
t testing.TB
js jetstream.JetStream
mu sync.Mutex
// rev is the revision's number; entries are its entries by module.
rev int64
entries map[string]meshcatalog.CatalogEntry
// src is the gate's source revision of catalogBranch, as its verb source answers it.
src meshdelivery.SourceAnswer
ctx context.Context
}
func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder {
t.Helper()
h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{
// the claimant of mesh-module-assigner, whose seat-holders view names the seats' holders (design 47 G §717, A-298)
claimantModule: {Module: claimantModule, ManifestDigest: "sha256:claimant", Manifest: map[string]any{"module": claimantModule,
"claims": []any{map[string]any{"name": "mesh-module-assigner", "scope": "mesh"}}}}}}
events, err := js.Stream(ctx, process.EventsStream)
if err != nil {
t.Fatal(err)
}
info := events.CachedInfo().Config
if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") {
info.Subjects = append(info.Subjects, "mesh.seat.*.event.>")
if _, err := js.UpdateStream(ctx, info); err != nil {
t.Fatal(err)
}
}
answer := func(m *nats.Msg) {
h.mu.Lock()
a := meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}}
for _, e := range h.entries {
a.Entries = append(a.Entries, e)
}
h.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
// The branch follower's other two first views (design 47 G §614): the registrations, each module on catalogBranch, and the
// gate's source, which keeps catalogBranch.
registrations := func(m *nats.Msg) {
h.mu.Lock()
a := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
for module := range h.entries {
a.Registrations = append(a.Registrations, meshcatalog.Registration{Module: module, Version: 1, Branches: []string{catalogBranch}})
}
h.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
// The gate's verb source: given no branch the branches it keeps, given catalogBranch its source revision (A-173).
source := func(m *nats.Msg) {
var in meshdelivery.SourceInput
_ = json.Unmarshal(m.Data, &in)
var a any = map[string]any{"branches": []string{catalogBranch}}
if in.Branch != "" {
h.mu.Lock()
a = meshdelivery.SourceAnswer{Branch: in.Branch, Revision: h.src.Revision, Group: h.src.Group, Repositories: h.src.Repositories}
h.mu.Unlock()
}
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": answer,
"mesh.seat.mesh-catalog.call.registrations": registrations, "mesh.seat.mesh-delivery.call.source": source} {
for _, s := range []string{subject + ".>", subject} {
sub, err := nc.Subscribe(s, handle)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
}
return h
}
// catalogBranch is the one watched branch of these tests: no branch is named in the view's code (design 47 G §532).
const catalogBranch = "release"
// moveSource is the gate's next source revision of catalogBranch, said as source-moved (design 47 G §507, A-173).
func (h *catalogHolder) moveSource(rev int64, group string, heads map[string]string) {
h.t.Helper()
h.mu.Lock()
h.src = meshdelivery.SourceAnswer{Revision: rev, Group: group}
for repository, head := range heads {
h.src.Repositories = append(h.src.Repositories, meshdelivery.SourceHead{Repository: repository, Head: head, Trunk: catalogBranch})
}
h.mu.Unlock()
body, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: catalogBranch, Revision: rev, Group: group})
if _, err := h.js.Publish(h.ctx, meshdelivery.SubjectSourceMoved(catalogBranch), body); err != nil {
h.t.Fatal(err)
}
}
// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision.
func (h *catalogHolder) put(e meshcatalog.CatalogEntry) {
h.t.Helper()
h.mu.Lock()
h.rev++
h.entries[e.Module] = e
rev := h.rev
h.mu.Unlock()
body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: catalogBranch, Revision: rev, Changed: []string{e.Module}})
if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision+"."+catalogBranch, body); err != nil {
h.t.Fatal(err)
}
}
// TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a
// broker file) against an embedded bus holding the three buckets, and asks its socket.
func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[b] = kv
}
put := func(bucket, key string, value []byte) {
t.Helper()
if _, err := kvs[bucket].Put(ctx, key, value); err != nil {
t.Fatal(err)
}
}
haveValue := func(build string, phase string, health string) []byte {
b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: build, Phase: phase, Health: health})
if err != nil {
t.Fatal(err)
}
return b
}
// The seat's assignments (issue 774, ADR 0549 §3): x and y on a, nothing on b.
place := func(node string, modules ...string) {
p := meshmoduleassigner.Assignments{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: m})
}
values, err := meshmoduleassignerclient.AssignmentsValues(p)
if err != nil {
t.Fatal(err)
}
for _, k := range slices.Sorted(maps.Keys(values)) { // a key per module, then the node key
put(view.AssignmentsBucket, k, values[k])
}
}
place("a", "x", "y")
place("b")
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseChecking, nodeengineclient.HealthWaits))
put(view.HaveBucket, "b.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
rowsMatch := func(check func(view.Snapshot) bool) view.Snapshot {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return s
}
}
}
t.Fatalf("the socket never showed the expected rows; last: %+v", last)
return last
}
byKey := func(s view.Snapshot, node, module string) view.Row {
for _, r := range s.Rows {
if r.Node == node && r.Module == module {
return r
}
}
return view.Row{}
}
s := rowsMatch(func(s view.Snapshot) bool {
return byKey(s, "a", "x").Health == "waits" && byKey(s, "a", "y").Waits == "no have yet" && byKey(s, "b", "x").Status == view.StatusLeftBehind
})
if s.Counted() != 2 {
t.Fatalf("counted = %d, want 2 (a.x, a.y): %+v", s.Counted(), s.Rows)
}
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
rowsMatch(func(s view.Snapshot) bool { r := byKey(s, "a", "x"); return r.Phase == "running" && r.Health == "ok" })
}
// TestProcessWholeShowsWhatTheWatchmanSays starts the program whole and writes an offline presence and a stuck key to the
// watchman's buckets, sees both shown, then writes them online and moved and sees both gone.
func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[b] = kv
}
presence := func(key string, online bool) {
t.Helper()
b, _ := meshwatchdogsclient.EncodeNodePresence(meshwatchdogsclient.NodePresence{Online: online})
if _, err := kvs[view.PresenceBucket].Put(ctx, key, b); err != nil {
t.Fatal(err)
}
}
stuck := func(moved bool) {
t.Helper()
b, _ := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: view.HaveBucket, Key: "a.x", Waited: "a report from a", Since: time.Now(), Moved: moved})
if _, err := kvs[view.StuckBucket].Put(ctx, view.HaveBucket+".a.x", b); err != nil {
t.Fatal(err)
}
}
presence("a", false)
presence("part.mesh-delivery-view.i1", false)
stuck(false)
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
wait := func(check func(view.Snapshot) bool) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("the socket never showed the expected lists; last: %+v", last)
}
wait(func(s view.Snapshot) bool { return len(s.Offline) == 2 && len(s.Stuck) == 1 && s.Stuck[0].Key == "a.x" })
presence("a", true)
presence("part.mesh-delivery-view.i1", true)
stuck(true)
wait(func(s view.Snapshot) bool { return len(s.Offline) == 0 && len(s.Stuck) == 0 })
}
// TestProcessWholeShowsTheIndexersBucketsOnceTheyExist starts the program whole before the indexer's buckets exist, then
// creates them and puts keys: the view's absent-bucket retry (follow, unchanged) finds them and the snapshot holds them.
// The retry is the view's own, every 5 s, so the wait here is 15 s, not 5.
func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 1241/AC5
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
t.Fatal(err)
}
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
catalog := serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
// The view is up and the indexer's buckets are not there.
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if _, err := view.Ask(socket); err == nil {
break
}
}
cands, err := js.KeyValue(ctx, view.CandidatesBucket)
if err != nil {
t.Fatal(err)
}
const head = "0123456789abcdef0123456789abcdef01234567"
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}})
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}})
if err != nil {
t.Fatal(err)
}
if _, err := cands.Put(ctx, "feat/x", cand); err != nil {
t.Fatal(err)
}
catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}})
var last view.Snapshot
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if len(s.Groups) == 1 && len(s.Fingerprints) == 1 {
if s.Groups[0].Group != "feat/x" || s.Fingerprints[0].Module != "a" || s.Fingerprints[0].State != view.FPNone {
t.Fatalf("snapshot = %+v", s)
}
return
}
}
}
t.Fatalf("the snapshot never held the indexer's keys; last: %+v", last)
}
// TestProcessWholeListsEveryDifference starts the program whole before the assigner's differences bucket exists, then
// creates it and writes a difference explained by a rule and one unexplained: the socket's snapshot lists both with their
// rule, and a difference taken away leaves the list (1297/B8: the view lists every difference with its rule).
func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
kv, err := js.KeyValue(ctx, view.WantsBucket)
if err != nil {
t.Fatal(err)
}
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
for key, d := range map[string]meshwantsclient.Want{
"want.ace.plex": {Fp: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
"want.g14.plex": {Fp: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it healthy yet"},
} {
v, err := meshwantsclient.EncodeWant(d)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, v); err != nil {
t.Fatal(err)
}
}
wait := func(check func(view.Snapshot) bool, what string) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("%s; last: %+v", what, last)
}
wait(func(s view.Snapshot) bool {
d := s.Wants
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == meshwantsclient.RuleCanary
}, "the two wants never showed with where they stand")
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
t.Fatal(err)
}
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
}
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
// answers and the teller's answers buckets, and a fixture hold ask (in release 5 the assigner writes none yet). The
// operator's answer over the socket is written under `<ask key>.view` in the view's bucket; the ask shows the choice, and
// leaves the list once the teller's answer is under its key. The merge's Approve is refused with its reason and writes
// nothing (1297/B9a AC).
func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 1297/B9a AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
bucket := func(name string, history uint8) jetstream.KeyValue {
t.Helper()
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history})
if err != nil {
t.Fatal(err)
}
return kv
}
askBucket := bucket(view.AsksBucket, view.AssignerAsksHistory)
viewBucket := bucket(viewstate.BucketsOf(testView).Answers, viewstate.ViewAnswersHistory)
bucket(testView+"_"+views.MarksState, 1)
teller := bucket(view.AnswersBucket, view.AnswersHistory)
// The view says its answers on its views family (design 47 G A-178), which the teller follows.
said := make(chan *nats.Msg, 8)
sub, err := nc.ChanSubscribe("mesh.mod."+testView+".event."+views.EventPrefix+viewstate.ViewAnswersState, said)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { sub.Unsubscribe() })
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
const fp = "src2:2222222222222222222222222222222222222222222222222222222222222222"
key, err := asks.HoldAskKey("plex", fp)
if err != nil {
t.Fatal(err)
}
ask := oc.Ask{ID: meshdeliveryclient.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: occ.Operator,
Expires: occ.FormatTime(time.Now().Add(time.Hour)), OnExpiry: "the fingerprint stays held",
Options: []oc.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: occ.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: occ.Approve}}}
value, _ := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask)})
if _, err := askBucket.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
wait := func(check func(view.Snapshot) bool, what string) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("%s; last: %+v", what, last)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].How == view.AnswerInTheView }, "the open ask never showed")
if err := view.AnswerAsk(socket, key, "release", "the check was a flake"); err != nil {
t.Fatal(err)
}
e, err := viewBucket.Get(ctx, key+".view")
if err != nil {
t.Fatalf("the view wrote no answer under %s.view: %v", key, err)
}
if a, err := viewstate.DecodeViewAnswer(e.Value()); err != nil || a.Option != "release" {
t.Fatalf("answer = %+v, %v", a, err)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].Chosen == "release" }, "the ask never showed the view's choice")
select {
case msg := <-said:
if !strings.Contains(string(msg.Data), key+".view") {
t.Errorf("the view said %s, not its answer under %s.view", msg.Data, key)
}
case <-time.After(20 * time.Second):
t.Error("the view never said its answer on views.answers")
}
// The merge's Approve is no ask of the view's (design 47 G §431): refused by its key, never written.
mkey, err := meshdeliveryclient.MergeAskKey("feat/x", []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567"}})
if err != nil {
t.Fatal(err)
}
if err := view.AnswerAsk(socket, mkey, "approve", ""); err == nil || !strings.Contains(err.Error(), "answer it on the phone (issue 953)") {
t.Errorf("merge Approve = %v, want the refusal naming the phone", err)
}
if _, err := viewBucket.Get(ctx, mkey+".view"); err == nil {
t.Error("the view wrote an answer to the merge's ask")
}
// The teller fills the ask: it leaves the list.
answer, err := meshdeliveryclient.EncodeAnswer(meshdeliveryclient.Answer{Option: "release", Warrant: ask.ID + "@2026-10-23T12:00:00Z", Words: "answered in the view"})
if err != nil {
t.Fatal(err)
}
if _, err := teller.Put(ctx, key, answer); err != nil {
t.Fatal(err)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 0 }, "the answered ask never left the list")
}
// TestProcessWholeShowsTheBacklogOfBuildAsks starts the program whole against a bus holding the go seat's work-queue stream
// with its three shared workers, three check asks waiting and one held by a node: the socket's backlog row names three
// waiting, one held, and the oldest's age (1887/A8a AC; compiled in the unit run, run by the merge check).
func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
stream, _ := buildstep.AskStream(buildstep.AskGo)
st, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat.node-build-go.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
var checkWorker jetstream.Consumer
for _, kind := range buildstep.AskKinds {
name, _ := buildstep.AskWorker(buildstep.AskGo, kind)
subject, _ := buildstep.AskSubject(buildstep.AskGo, kind)
c, err := st.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: name, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
if kind == buildstep.AskCheck {
checkWorker = c
}
}
subject, _ := buildstep.AskSubject(buildstep.AskGo, buildstep.AskCheck)
oldest := time.Now().UTC().Add(-2 * time.Minute)
for i, id := range []string{"m0000001aaaaaa", "m0000002bbbbbb", "m0000003cccccc", "m0000004dddddd"} {
body, err := buildstep.EncodeStepAsk(buildstep.StepAsk{ID: id, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: "mesh-delivery-checker", At: oldest.Add(time.Duration(i) * time.Second),
Check: &buildstep.CheckJob{Fetch: buildstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}, Levels: []string{buildstep.LevelUnit}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, subject, body); err != nil {
t.Fatal(err)
}
}
// A node pulls one and holds it unacknowledged.
held, err := checkWorker.Fetch(1)
if err != nil {
t.Fatal(err)
}
for range held.Messages() {
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
s, err := view.Ask(socket)
if err != nil {
continue
}
last = s
for _, b := range s.Backlog {
if b.Language == "go" && b.Kind == "check" && b.Waiting == 3 && b.Held == 1 && b.OldestSecs >= 118 {
return
}
}
}
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
}
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{view.WantsBucket, view.HaveBucket, viewstate.BucketsOf(testView).Reach} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
catalog := serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
fp := "src2:" + strings.Repeat("2", 64)
want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
if _, err := kvs[view.WantsBucket].Put(ctx, "want.ace.web", want); err != nil {
t.Fatal(err)
}
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
t.Fatal(err)
}
// The gate's source of the watched branch moves: the view asks the verb source for it and reads no gate state (A-173).
const rev = 41
catalog.moveSource(rev, "feat/x", map[string]string{"novox/web": strings.Repeat("1", 40)})
time.Sleep(time.Second)
if info, err := events.Info(ctx, jetstream.WithSubjectFilter("mesh.mod.mesh-delivery-view.event."+DeliveredEventType)); err != nil || len(info.State.Subjects) != 0 {
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
}
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", good); err != nil {
t.Fatal(err)
}
var msg *jetstream.RawStreamMsg
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
}
if msg == nil {
t.Fatal("the delivery of the source revision was never said")
}
var e view.DeliveredEvent
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Branch != catalogBranch || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
t.Errorf("the event = %+v, %v; want the branch %s, revision %d, group feat/x and the heads", e, err, catalogBranch, rev)
}
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
t.Errorf("the headers = %v", msg.Header)
}
// The reach it wrote is said as the event reached, with the reach as written (ADR 0596 §2).
var rm *jetstream.RawStreamMsg
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && rm == nil; time.Sleep(100 * time.Millisecond) {
rm, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+ReachedEventType)
}
if rm == nil {
t.Fatal("the reach of novox/web was written and never said as the event reached")
}
var re ReachedEvent
if err := json.Unmarshal(rm.Data, &re); err != nil || re.Key != "novox/web" {
t.Fatalf("the event reached = %s, %v", rm.Data, err)
}
if stored, gerr := kvs[viewstate.BucketsOf(testView).Reach].Get(ctx, re.Key); gerr != nil || string(stored.Value()) != string(re.Reach) {
t.Errorf("the event carries %s, the state reach holds %v (%v)", re.Reach, stored, gerr)
}
}
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
// and composition.
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{view.WantsBucket, view.HaveBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
time.Sleep(2 * time.Second) // the view has read what stood: nothing
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
digest := "sha256:" + strings.Repeat("c", 64)
put := func(bucket, key string, v []byte, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
t.Fatal(err)
}
}
want := func(node, module string) {
v, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Composed: digest})
put(view.WantsBucket, "want."+node+"."+module, v, err)
}
have := func(node, module, fp string, phase string, health string, detail string) {
v, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Health: health, Detail: detail})
put(view.HaveBucket, node+"."+module, v, err)
}
have("ace", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
want("ace", "web")
have("g14", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
want("g14", "web")
said := func() map[string]jetstream.RawStreamMsg {
out := map[string]jetstream.RawStreamMsg{}
for seq := uint64(1); ; seq++ {
m, err := events.GetMsg(ctx, seq)
if err != nil {
return out
}
out[m.Header.Get("x-event-id")] = *m
}
}
waitFor := func(id string) jetstream.RawStreamMsg {
t.Helper()
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if m, ok := said()[id]; ok {
return m
}
}
t.Fatalf("never said %s; said %v", id, func() []string {
var ids []string
for k := range said() {
ids = append(ids, k)
}
return ids
}())
return jetstream.RawStreamMsg{}
}
hex := strings.Repeat("c", 64)
mv := waitFor("moving.ace.web." + hex)
var body view.MotionBody
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
t.Errorf("moving = %+v, %v", body, err)
}
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
t.Errorf("subject = %s", mv.Subject)
}
have("ace", "web", fpNew, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
st := waitFor("settled.ace.web." + hex)
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
t.Errorf("settled = %+v, %v", body, err)
}
have("g14", "web", fpOld, nodeengineclient.PhaseHeld, nodeengineclient.HealthWaits, "no space left")
stk := waitFor("stuck.g14.web." + hex)
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
t.Errorf("stuck = %+v, %v", body, err)
}
time.Sleep(time.Second)
count := map[string]int{}
for id := range said() {
count[strings.SplitN(id, ".", 2)[0]]++
}
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
}
}