Files
mesh-delivery/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go
T
jschoubben da8384d10e VW5: the view takes each watched branch's source from the gate's verb source, per branch
The view read the gate's state source and its history, fixed to the branch main. Each watched branch's source is now
the seat mesh-delivery's verb source given that branch, asked again on its source-moved; the source row and the event
delivered are per branch, a repository's reach is from the source of its modules' registration's first branch, laid
over the reach the view's own state kept at its start, and view_tiers given no revision answers per branch. The
manifest drops its read of mesh-delivery-gate.source and names the source answer's fields it relies on.

Builds: design 47 G §507, §512, §532, §558, A-173, A-225

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 06:21:49 +01:00

1001 lines
41 KiB
Go

//go:build integration
package main
import (
"context"
"encoding/json"
"io"
"maps"
"mesh-delivery-view/internal/view"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// serveOwners serves the six states of the indexer and the assigner the view's process follows by their first views
// (ADR 0615 §1, DLV-M4): each owner's read tool and views family, with the view's durable consumer (`n_mesh-delivery-view`).
// A bucket the test made is kept; the others are made empty.
func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) {
t.Helper()
for _, o := range ownersFollowed {
viewstest.Serve(t, ctx, js, nc, o.owner, o.state, o.tool, "n_"+viewstate.ViewModule)
}
}
// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the
// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the
// view's durable consumer hears.
type catalogHolder struct {
t testing.TB
js jetstream.JetStream
mu sync.Mutex
// rev is the revision's number; entries are its entries by module.
rev int64
entries map[string]meshcatalog.CatalogEntry
// src is the gate's source revision of catalogBranch, as its verb source answers it.
src meshdelivery.SourceAnswer
ctx context.Context
}
func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder {
t.Helper()
h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{}}
events, err := js.Stream(ctx, process.EventsStream)
if err != nil {
t.Fatal(err)
}
info := events.CachedInfo().Config
if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") {
info.Subjects = append(info.Subjects, "mesh.seat.*.event.>")
if _, err := js.UpdateStream(ctx, info); err != nil {
t.Fatal(err)
}
}
answer := func(m *nats.Msg) {
h.mu.Lock()
a := meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}}
for _, e := range h.entries {
a.Entries = append(a.Entries, e)
}
h.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
// The branch follower's other two first views (design 47 G §614): the registrations, each module on catalogBranch, and the
// gate's source, which keeps catalogBranch.
registrations := func(m *nats.Msg) {
h.mu.Lock()
a := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
for module := range h.entries {
a.Registrations = append(a.Registrations, meshcatalog.Registration{Module: module, Version: 1, Branches: []string{catalogBranch}})
}
h.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
// The gate's verb source: given no branch the branches it keeps, given catalogBranch its source revision (A-173).
source := func(m *nats.Msg) {
var in meshdelivery.SourceInput
_ = json.Unmarshal(m.Data, &in)
var a any = map[string]any{"branches": []string{catalogBranch}}
if in.Branch != "" {
h.mu.Lock()
a = meshdelivery.SourceAnswer{Branch: in.Branch, Revision: h.src.Revision, Group: h.src.Group, Repositories: h.src.Repositories}
h.mu.Unlock()
}
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
}
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": answer,
"mesh.seat.mesh-catalog.call.registrations": registrations, "mesh.seat.mesh-delivery.call.source": source} {
for _, s := range []string{subject + ".>", subject} {
sub, err := nc.Subscribe(s, handle)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
}
return h
}
// catalogBranch is the one watched branch of these tests: no branch is named in the view's code (design 47 G §532).
const catalogBranch = "release"
// moveSource is the gate's next source revision of catalogBranch, said as source-moved (design 47 G §507, A-173).
func (h *catalogHolder) moveSource(rev int64, group string, heads map[string]string) {
h.t.Helper()
h.mu.Lock()
h.src = meshdelivery.SourceAnswer{Revision: rev, Group: group}
for repository, head := range heads {
h.src.Repositories = append(h.src.Repositories, meshdelivery.SourceHead{Repository: repository, Head: head, Trunk: catalogBranch})
}
h.mu.Unlock()
body, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: catalogBranch, Revision: rev, Group: group})
if _, err := h.js.Publish(h.ctx, meshdelivery.SubjectSourceMoved(catalogBranch), body); err != nil {
h.t.Fatal(err)
}
}
// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision.
func (h *catalogHolder) put(e meshcatalog.CatalogEntry) {
h.t.Helper()
h.mu.Lock()
h.rev++
h.entries[e.Module] = e
rev := h.rev
h.mu.Unlock()
body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: catalogBranch, Revision: rev, Changed: []string{e.Module}})
if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision+"."+catalogBranch, body); err != nil {
h.t.Fatal(err)
}
}
// TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a
// broker file) against an embedded bus holding the three buckets, and asks its socket.
func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[b] = kv
}
put := func(bucket, key string, value []byte) {
t.Helper()
if _, err := kvs[bucket].Put(ctx, key, value); err != nil {
t.Fatal(err)
}
}
haveValue := func(build string, phase string, health string) []byte {
b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: build, Phase: phase, Health: health})
if err != nil {
t.Fatal(err)
}
return b
}
// The seat's assignments (issue 774, ADR 0549 §3): x and y on a, nothing on b.
place := func(node string, modules ...string) {
p := meshmoduleassigner.Assignments{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: m})
}
values, err := meshmoduleassignerclient.AssignmentsValues(p)
if err != nil {
t.Fatal(err)
}
for _, k := range slices.Sorted(maps.Keys(values)) { // a key per module, then the node key
put(view.AssignmentsBucket, k, values[k])
}
}
place("a", "x", "y")
place("b")
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseChecking, nodeengineclient.HealthWaits))
put(view.HaveBucket, "b.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
rowsMatch := func(check func(view.Snapshot) bool) view.Snapshot {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return s
}
}
}
t.Fatalf("the socket never showed the expected rows; last: %+v", last)
return last
}
byKey := func(s view.Snapshot, node, module string) view.Row {
for _, r := range s.Rows {
if r.Node == node && r.Module == module {
return r
}
}
return view.Row{}
}
s := rowsMatch(func(s view.Snapshot) bool {
return byKey(s, "a", "x").Health == "waits" && byKey(s, "a", "y").Waits == "no have yet" && byKey(s, "b", "x").Status == view.StatusLeftBehind
})
if s.Counted() != 2 {
t.Fatalf("counted = %d, want 2 (a.x, a.y): %+v", s.Counted(), s.Rows)
}
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
rowsMatch(func(s view.Snapshot) bool { r := byKey(s, "a", "x"); return r.Phase == "running" && r.Health == "ok" })
}
// TestProcessWholeShowsWhatTheWatchmanSays starts the program whole and writes an offline presence and a stuck key to the
// watchman's buckets, sees both shown, then writes them online and moved and sees both gone.
func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[b] = kv
}
presence := func(key string, online bool) {
t.Helper()
b, _ := meshwatchdogsclient.EncodeNodePresence(meshwatchdogsclient.NodePresence{Online: online})
if _, err := kvs[view.PresenceBucket].Put(ctx, key, b); err != nil {
t.Fatal(err)
}
}
stuck := func(moved bool) {
t.Helper()
b, _ := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: view.HaveBucket, Key: "a.x", Waited: "a report from a", Since: time.Now(), Moved: moved})
if _, err := kvs[view.StuckBucket].Put(ctx, view.HaveBucket+".a.x", b); err != nil {
t.Fatal(err)
}
}
presence("a", false)
presence("part.mesh-delivery-view.i1", false)
stuck(false)
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
wait := func(check func(view.Snapshot) bool) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("the socket never showed the expected lists; last: %+v", last)
}
wait(func(s view.Snapshot) bool { return len(s.Offline) == 2 && len(s.Stuck) == 1 && s.Stuck[0].Key == "a.x" })
presence("a", true)
presence("part.mesh-delivery-view.i1", true)
stuck(true)
wait(func(s view.Snapshot) bool { return len(s.Offline) == 0 && len(s.Stuck) == 0 })
}
// TestProcessWholeShowsTheIndexersBucketsOnceTheyExist starts the program whole before the indexer's buckets exist, then
// creates them and puts keys: the view's absent-bucket retry (follow, unchanged) finds them and the snapshot holds them.
// The retry is the view's own, every 5 s, so the wait here is 15 s, not 5.
func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 1241/AC5
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
t.Fatal(err)
}
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
catalog := serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
// The view is up and the indexer's buckets are not there.
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if _, err := view.Ask(socket); err == nil {
break
}
}
cands, err := js.KeyValue(ctx, view.CandidatesBucket)
if err != nil {
t.Fatal(err)
}
const head = "0123456789abcdef0123456789abcdef01234567"
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}})
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}})
if err != nil {
t.Fatal(err)
}
if _, err := cands.Put(ctx, "feat/x", cand); err != nil {
t.Fatal(err)
}
catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}})
var last view.Snapshot
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if len(s.Groups) == 1 && len(s.Fingerprints) == 1 {
if s.Groups[0].Group != "feat/x" || s.Fingerprints[0].Module != "a" || s.Fingerprints[0].State != view.FPNone {
t.Fatalf("snapshot = %+v", s)
}
return
}
}
}
t.Fatalf("the snapshot never held the indexer's keys; last: %+v", last)
}
// TestProcessWholeListsEveryDifference starts the program whole before the assigner's differences bucket exists, then
// creates it and writes a difference explained by a rule and one unexplained: the socket's snapshot lists both with their
// rule, and a difference taken away leaves the list (1297/B8: the view lists every difference with its rule).
func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
kv, err := js.KeyValue(ctx, view.AssignerWantsBucket)
if err != nil {
t.Fatal(err)
}
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
for key, d := range map[string]meshwantsclient.Want{
"want.ace.plex": {Fp: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
"want.g14.plex": {Fp: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it healthy yet"},
} {
v, err := meshwantsclient.EncodeWant(d)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, v); err != nil {
t.Fatal(err)
}
}
wait := func(check func(view.Snapshot) bool, what string) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("%s; last: %+v", what, last)
}
wait(func(s view.Snapshot) bool {
d := s.Wants
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == meshwantsclient.RuleCanary
}, "the two wants never showed with where they stand")
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
t.Fatal(err)
}
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
}
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
// answers and the teller's answers buckets, and a fixture hold ask (in release 5 the assigner writes none yet). The
// operator's answer over the socket is written under `<ask key>.view` in the view's bucket; the ask shows the choice, and
// leaves the list once the teller's answer is under its key. The merge's Approve is refused with its reason and writes
// nothing (1297/B9a AC).
func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 1297/B9a AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
bucket := func(name string, history uint8) jetstream.KeyValue {
t.Helper()
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history})
if err != nil {
t.Fatal(err)
}
return kv
}
askBucket := bucket(view.AssignerAsksBucket, view.AssignerAsksHistory)
viewBucket := bucket(viewstate.ViewAnswersBucket, viewstate.ViewAnswersHistory)
bucket(viewstate.ViewModule+"_"+views.MarksState, 1)
teller := bucket(view.AnswersBucket, view.AnswersHistory)
// The view says its answers on its views family (design 47 G A-178), which the teller follows.
said := make(chan *nats.Msg, 8)
sub, err := nc.ChanSubscribe("mesh.mod."+viewstate.ViewModule+".event."+views.EventPrefix+viewstate.ViewAnswersState, said)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { sub.Unsubscribe() })
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
const fp = "src2:2222222222222222222222222222222222222222222222222222222222222222"
key, err := asks.HoldAskKey("plex", fp)
if err != nil {
t.Fatal(err)
}
ask := oc.Ask{ID: meshdeliveryclient.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: occ.Operator,
Expires: occ.FormatTime(time.Now().Add(time.Hour)), OnExpiry: "the fingerprint stays held",
Options: []oc.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: occ.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: occ.Approve}}}
value, _ := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask)})
if _, err := askBucket.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
wait := func(check func(view.Snapshot) bool, what string) {
t.Helper()
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if s, err := view.Ask(socket); err == nil {
last = s
if check(s) {
return
}
}
}
t.Fatalf("%s; last: %+v", what, last)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].How == view.AnswerInTheView }, "the open ask never showed")
if err := view.AnswerAsk(socket, key, "release", "the check was a flake"); err != nil {
t.Fatal(err)
}
e, err := viewBucket.Get(ctx, key+".view")
if err != nil {
t.Fatalf("the view wrote no answer under %s.view: %v", key, err)
}
if a, err := viewstate.DecodeViewAnswer(e.Value()); err != nil || a.Option != "release" {
t.Fatalf("answer = %+v, %v", a, err)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].Chosen == "release" }, "the ask never showed the view's choice")
select {
case msg := <-said:
if !strings.Contains(string(msg.Data), key+".view") {
t.Errorf("the view said %s, not its answer under %s.view", msg.Data, key)
}
case <-time.After(20 * time.Second):
t.Error("the view never said its answer on views.answers")
}
// The merge's Approve is no ask of the view's (design 47 G §431): refused by its key, never written.
mkey, err := meshdeliveryclient.MergeAskKey("feat/x", []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567"}})
if err != nil {
t.Fatal(err)
}
if err := view.AnswerAsk(socket, mkey, "approve", ""); err == nil || !strings.Contains(err.Error(), "answer it on the phone (issue 953)") {
t.Errorf("merge Approve = %v, want the refusal naming the phone", err)
}
if _, err := viewBucket.Get(ctx, mkey+".view"); err == nil {
t.Error("the view wrote an answer to the merge's ask")
}
// The teller fills the ask: it leaves the list.
answer, err := meshdeliveryclient.EncodeAnswer(meshdeliveryclient.Answer{Option: "release", Warrant: ask.ID + "@2026-10-23T12:00:00Z", Words: "answered in the view"})
if err != nil {
t.Fatal(err)
}
if _, err := teller.Put(ctx, key, answer); err != nil {
t.Fatal(err)
}
wait(func(s view.Snapshot) bool { return len(s.Asks) == 0 }, "the answered ask never left the list")
}
// TestProcessWholeShowsTheBacklogOfBuildAsks starts the program whole against a bus holding the go seat's work-queue stream
// with its three shared workers, three check asks waiting and one held by a node: the socket's backlog row names three
// waiting, one held, and the oldest's age (1887/A8a AC; compiled in the unit run, run by the merge check).
func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
stream, _ := buildstep.AskStream(buildstep.AskGo)
st, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat.node-build-go.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
var checkWorker jetstream.Consumer
for _, kind := range buildstep.AskKinds {
name, _ := buildstep.AskWorker(buildstep.AskGo, kind)
subject, _ := buildstep.AskSubject(buildstep.AskGo, kind)
c, err := st.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: name, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
if kind == buildstep.AskCheck {
checkWorker = c
}
}
subject, _ := buildstep.AskSubject(buildstep.AskGo, buildstep.AskCheck)
oldest := time.Now().UTC().Add(-2 * time.Minute)
for i, id := range []string{"m0000001aaaaaa", "m0000002bbbbbb", "m0000003cccccc", "m0000004dddddd"} {
body, err := buildstep.EncodeStepAsk(buildstep.StepAsk{ID: id, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: "mesh-delivery-checker", At: oldest.Add(time.Duration(i) * time.Second),
Check: &buildstep.CheckJob{Fetch: buildstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}, Levels: []string{buildstep.LevelUnit}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, subject, body); err != nil {
t.Fatal(err)
}
}
// A node pulls one and holds it unacknowledged.
held, err := checkWorker.Fetch(1)
if err != nil {
t.Fatal(err)
}
for range held.Messages() {
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "run", "cli.sock")
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
var last view.Snapshot
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
s, err := view.Ask(socket)
if err != nil {
continue
}
last = s
for _, b := range s.Backlog {
if b.Language == "go" && b.Kind == "check" && b.Waiting == 3 && b.Held == 1 && b.OldestSecs >= 118 {
return
}
}
}
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
}
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{view.AssignerWantsBucket, view.HaveBucket, viewstate.ReachBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
catalog := serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
fp := "src2:" + strings.Repeat("2", 64)
want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
if _, err := kvs[view.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil {
t.Fatal(err)
}
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
t.Fatal(err)
}
// The gate's source of the watched branch moves: the view asks the verb source for it and reads no gate state (A-173).
const rev = 41
catalog.moveSource(rev, "feat/x", map[string]string{"novox/web": strings.Repeat("1", 40)})
time.Sleep(time.Second)
if info, err := events.Info(ctx, jetstream.WithSubjectFilter("mesh.mod.mesh-delivery-view.event."+DeliveredEventType)); err != nil || len(info.State.Subjects) != 0 {
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
}
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", good); err != nil {
t.Fatal(err)
}
var msg *jetstream.RawStreamMsg
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
}
if msg == nil {
t.Fatal("the delivery of the source revision was never said")
}
var e view.DeliveredEvent
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Branch != catalogBranch || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
t.Errorf("the event = %+v, %v; want the branch %s, revision %d, group feat/x and the heads", e, err, catalogBranch, rev)
}
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
t.Errorf("the headers = %v", msg.Header)
}
// The reach it wrote is said as the event reached, with the reach as written (ADR 0596 §2).
var rm *jetstream.RawStreamMsg
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && rm == nil; time.Sleep(100 * time.Millisecond) {
rm, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+ReachedEventType)
}
if rm == nil {
t.Fatal("the reach of novox/web was written and never said as the event reached")
}
var re ReachedEvent
if err := json.Unmarshal(rm.Data, &re); err != nil || re.Key != "novox/web" {
t.Fatalf("the event reached = %s, %v", rm.Data, err)
}
if stored, gerr := kvs[viewstate.ReachBucket].Get(ctx, re.Key); gerr != nil || string(stored.Value()) != string(re.Reach) {
t.Errorf("the event carries %s, the state reach holds %v (%v)", re.Reach, stored, gerr)
}
}
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
// and composition.
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{view.AssignerWantsBucket, view.HaveBucket, view.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
serveOwners(t, ctx, js, nc)
serveCatalog(t, ctx, js, nc)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
time.Sleep(2 * time.Second) // the view has read what stood: nothing
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
digest := "sha256:" + strings.Repeat("c", 64)
put := func(bucket, key string, v []byte, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
t.Fatal(err)
}
}
want := func(node, module string) {
v, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Composed: digest})
put(view.AssignerWantsBucket, "want."+node+"."+module, v, err)
}
have := func(node, module, fp string, phase string, health string, detail string) {
v, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Health: health, Detail: detail})
put(view.HaveBucket, node+"."+module, v, err)
}
have("ace", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
want("ace", "web")
have("g14", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
want("g14", "web")
said := func() map[string]jetstream.RawStreamMsg {
out := map[string]jetstream.RawStreamMsg{}
for seq := uint64(1); ; seq++ {
m, err := events.GetMsg(ctx, seq)
if err != nil {
return out
}
out[m.Header.Get("x-event-id")] = *m
}
}
waitFor := func(id string) jetstream.RawStreamMsg {
t.Helper()
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if m, ok := said()[id]; ok {
return m
}
}
t.Fatalf("never said %s; said %v", id, func() []string {
var ids []string
for k := range said() {
ids = append(ids, k)
}
return ids
}())
return jetstream.RawStreamMsg{}
}
hex := strings.Repeat("c", 64)
mv := waitFor("moving.ace.web." + hex)
var body view.MotionBody
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
t.Errorf("moving = %+v, %v", body, err)
}
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
t.Errorf("subject = %s", mv.Subject)
}
have("ace", "web", fpNew, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
st := waitFor("settled.ace.web." + hex)
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
t.Errorf("settled = %+v, %v", body, err)
}
have("g14", "web", fpOld, nodeengineclient.PhaseHeld, nodeengineclient.HealthWaits, "no space left")
stk := waitFor("stuck.g14.web." + hex)
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
t.Errorf("stuck = %+v, %v", body, err)
}
time.Sleep(time.Second)
count := map[string]int{}
for id := range said() {
count[strings.SplitN(id, ".", 2)[0]]++
}
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
}
}