From 0474a2a3cb5dc90266add5d1dea15b070f904ac7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 02:21:36 +0100 Subject: [PATCH] Keep the node's own composition's sealed values in its key and read a policy within the agent's context, so a restart applies nothing again (issue 1126) --- cmd/mesh-host/nodeagent.go | 14 +++++++- go.mod | 2 +- go.sum | 4 +++ internal/agent/agent.go | 12 ++++--- internal/agent/agent_test.go | 21 ++++++++++-- internal/link/have.go | 42 ++++++++++++++++++++++-- internal/link/have_test.go | 9 +++-- internal/selfcompose/selfcompose.go | 12 +++---- internal/selfcompose/selfcompose_test.go | 14 +++++--- 9 files changed, 104 insertions(+), 26 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 44d250e..b4e16bd 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -56,6 +56,14 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up type sessionHaves struct{ session *link.Session } func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) { + if module == agent.NodeModule { + // The node's own key: the digest of its own composition it runs, and what that carries sealed. + got, ok := h.session.ReadNodeHave(ctx) + if !ok || got.Composed == "" { + return agent.Have{}, false + } + return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true + } got, ok := h.session.ReadHave(ctx, module) if !ok { return agent.Have{}, false @@ -68,7 +76,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) { if x.Module == agent.NodeModule { // The node's own key holds its facts and the digest of the composition it runs. if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) { - h.session.NoteNode(ctx, x.Composed) + h.session.NoteNode(ctx, x.Composed, x.Sealed) } return } @@ -511,6 +519,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess unseal: sealOpener(opts.state), // And what each module's have says it runs sealed: after a restart the agent has applied nothing yet. held: func(module string) []string { + if module == agent.NodeModule { + got, _ := session.ReadNodeHave(linked) + return got.Sealed + } got, ok := session.ReadHave(linked, module) if !ok { return nil diff --git a/go.mod b/go.mod index be6daa0..6c08518 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index e8660dd..25c0557 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/agent/agent.go b/internal/agent/agent.go index c49c539..dd101be 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -135,7 +135,7 @@ type Env struct { // Policy is a module's upgrade policy as its manifest derives it (catalogue.DerivedUpgrade); with SelfComposed, a // move of a module of policy record that recreates a container or moves its data waits for a person's yes in its // want (ADR 0547 §3). Nil judges no move. - Policy func(module string) string + Policy func(ctx context.Context, module string) string } // transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has @@ -377,8 +377,8 @@ func (a *Agent) NotComposed(ctx context.Context, why string) { } } -// writeHave writes a module's have; on a node composing its own, with the sealed values of the composition it names -// when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else +// writeHave writes a module's have (the node's own key for NodeModule); on a node composing its own, with the sealed +// values of the composition it names when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else // those the module's last have said at that digest. // // A writer that names no composition (one written from the agent's running entry before this agent ran anything of the @@ -394,7 +394,9 @@ func (a *Agent) writeHave(ctx context.Context, h Have) { } } } - if a.env.SelfComposed && h.Sealed == nil && h.Module != NodeModule && h.Composed != "" { + // The node's own composition too: its core boxes are sealed again by the vault at a restart, and kept in its key the + // composer reuses them, so the restart applies nothing again (review of L2, round 3, finding 1). + if a.env.SelfComposed && h.Sealed == nil && h.Composed != "" { h.Sealed = a.sealedOf(ctx, h.Module, h.FP, h.Composed) } a.env.Haves.Write(ctx, h) @@ -1101,7 +1103,7 @@ func short(s string) string { // for this composition; the have says phase waiting-yes with the judgement, and the assigner asks a person. A composition // equal to what runs is no move. True when the module waits. func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp compNow) bool { - if module == NodeModule || a.env.Policy == nil || a.env.Policy(module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig { + if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig { return false } a.mu.Lock() diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 44290f5..ed10ab9 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -964,7 +964,7 @@ func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) { func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(t *testing.T) { b := newBed(t) b.env.SelfComposed, b.env.Trusted = true, nil - b.env.Policy = func(m string) string { + b.env.Policy = func(_ context.Context, m string) string { if m == "db" || m == "rec" { return catalogue.PolicyRecord } @@ -1001,6 +1001,21 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( // The same composition handed again is no move. b.comps.put(ck, craw) time.Sleep(50 * time.Millisecond) + // The node's own composition carries its sealed values in its key, as a module's have does, so a restart reuses its + // core boxes and applies nothing again (review of L2, round 3, finding 1). + box, _ := json.Marshal(map[string]any{"id": "core-box", "type": "file", "path": "/tmp/r6/core-box", "sealed": "core"}) + own := delivery.Composed{Node: node, Resources: []json.RawMessage{box}} + nk, _ := delivery.ComposedNodeKey(node) + nraw, _ := delivery.EncodeComposed(own) + ndig, _ := delivery.ComposedDigestOf(own) + b.comps.put(nk, nraw) + nwk, _ := delivery.WantNodeKey(node) + nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true}) + b.wants.put(nwk, nwraw) + eventually(t, "the node's own key does not carry its sealed values", func() bool { + h := b.haves.last(NodeModule) + return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed[0] == "core" + }) // The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still // names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the // agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said. @@ -1035,7 +1050,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( // A revision at the same build (a config value changed) is applied. _, craw2, dig2 := unsigned("web", fp, file("web", "x", "2")) b.comps.put(ck, craw2) - eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 2 && b.haves.last("web").Composed == dig2 }) + eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 3 && b.haves.last("web").Composed == dig2 }) // A record module: its first composition is no move; a new image recreates its container and waits for a yes. container := func(image string) json.RawMessage { @@ -1046,7 +1061,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64))) b.comps.put(dk, draw) want("db", fp, "") - eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 3 }) + eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 }) fpB := fpOf("b") dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64))) // rec is a record module this agent has run nothing of since its start (the switch and every delivery of the diff --git a/internal/link/have.go b/internal/link/have.go index f2b1931..50211f6 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -47,6 +47,9 @@ type haveState struct { node *delivery.HaveNode // nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts. nodeComposed string + // nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3, + // issue 1126): kept in its key so a restart reuses them and applies nothing again. + nodeSealed []string // written is the build of each module this node-engine wrote a have key for and that the declaration it applied // still holds; removed is each such module the latest declaration no longer holds, with the build its key last // said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted). @@ -416,6 +419,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { var raw []byte facts := *s.node facts.Composed = s.nodeComposed + facts.Sealed = s.nodeSealed if raw, err = delivery.EncodeHaveNode(facts); err == nil { want := facts if !q.putHave(write, bucket, key, raw, func(current []byte) bool { @@ -703,12 +707,14 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) { q.writeHaves(ctx, bus) } -// NoteNode takes the digest of the node's own composition it runs, and writes the node's key where it differs. -func (q *Session) NoteNode(ctx context.Context, composed string) { +// NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key +// (nil for a composition the node did not make itself), and writes the node's key where it differs. +func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string) { q.init() s := &q.haves s.mu.Lock() s.nodeComposed = composed + s.nodeSealed = sealed if s.node == nil { n := delivery.HaveNode{V: delivery.Version} s.node = &n @@ -720,6 +726,38 @@ func (q *Session) NoteNode(ctx context.Context, composed string) { q.writeHaves(ctx, bus) } +// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition +// it last ran is and carries sealed, read after a restart (ADR 0547 §3). +func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + hb, ok := bus.(HaveBus) + if !ok || bus == nil { + return delivery.HaveNode{}, false + } + key, err := delivery.HaveNodeKey(q.Membership.Node) + if err != nil { + return delivery.HaveNode{}, false + } + read, cancel := context.WithTimeout(ctx, q.timeout()) + defer cancel() + bucket, err := hb.HaveBind(read) + if err != nil { + return delivery.HaveNode{}, false + } + value, _, err := bucket.Get(read, key) + if err != nil { + return delivery.HaveNode{}, false + } + n, err := delivery.DecodeHaveNode(value) + if err != nil { + return delivery.HaveNode{}, false + } + return n, true +} + // ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot // be read answers as none: the agent then compares with the declaration it kept, and applies only what differs. func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index fdb66ca..ee263ac 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -582,9 +582,12 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved { t.Fatalf("removed: %+v", got) } - q.NoteNode(context.Background(), dig) - if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig { - t.Fatalf("the node key: %+v %v", n, err) + q.NoteNode(context.Background(), dig, []string{"core"}) + if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.Sealed) != 1 || n.Sealed[0] != "core" { + t.Fatalf("the node key, with its own composition's sealed values: %+v %v", n, err) + } + if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 { + t.Fatalf("the node key read back: %+v %v", n, ok) } } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 19a4d9c..12ef2a7 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -453,11 +453,10 @@ func (c *Composer) runningBox(box string) string { } if c.env.Held != nil { c.mu.Lock() + // The node's own key among them (agent.NodeModule): its core boxes, as its last run said them. modules := make([]string, 0, len(c.wants)) for m := range c.wants { - if m != agent.NodeModule { - modules = append(modules, m) - } + modules = append(modules, m) } c.mu.Unlock() sort.Strings(modules) @@ -514,15 +513,16 @@ func (c *Composer) builtManifest(ctx context.Context, module, fp string) (string } // Policy is a module's upgrade policy as its manifest at the indexer's trunk derives it (catalogue.DerivedUpgrade), for -// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read. -func (c *Composer) Policy(module string) string { +// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read. The manifest is +// read within ctx, the agent's (review of L2, round 1, finding 5), so a stopping agent stops the read. +func (c *Composer) Policy(ctx context.Context, module string) string { c.mu.Lock() m, ok := c.modules[module] c.mu.Unlock() if !ok || m.Manifest == "" { return "" } - raw, err := reader{c: c}.Manifest(context.Background(), m.Manifest) + raw, err := reader{c: c}.Manifest(ctx, m.Manifest) if err != nil { return "" } diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 8ade9a3..a7d220d 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -237,14 +237,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } return sealed[0] } + // And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and + // kept after a restart as a module's is, so the node's own composition does not move either. for _, row := range []struct { - name, held string - kept bool - }{{"the have holds the rotated value", boxOf(string(rotated.Value)), true}, {"the have holds the value before the rotation", boxOf(before), false}} { + name, held, in string + kept bool + }{{"the have holds the rotated value", boxOf(string(rotated.Value)), "keeper", true}, + {"the have holds the value before the rotation", boxOf(before), "keeper", false}, + {"the node's own key holds the rotated value", boxOf(string(rotated.Value)), agent.NodeModule, true}} { restarted := env restarted.Running = func() []delivery.Composed { return nil } restarted.Held = func(module string) []string { - if module == "keeper" { + if module == row.in { return []string{row.held} } return nil @@ -258,7 +262,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return nil, fmt.Errorf("not sealed to n1: %s", box) } again := New(restarted) - again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}} + again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}, agent.NodeModule: {}} rows, err := reader{c: again}.OwnSecrets(ctx, "n1") if err != nil || len(rows) != 1 { t.Fatalf("%s: own secrets %+v, %v", row.name, rows, err)