Keep the node's own composition's sealed values in its key and read a policy within the agent's context, so a restart applies nothing again (issue 1126)

cmd/mesh-host/nodeagent.go

@@ -56,6 +56,14 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
 type sessionHaves struct{ session *link.Session }
 
 func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
+	if module == agent.NodeModule {
+		// The node's own key: the digest of its own composition it runs, and what that carries sealed.
+		got, ok := h.session.ReadNodeHave(ctx)
+		if !ok || got.Composed == "" {
+			return agent.Have{}, false
+		}
+		return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true
+	}
 	got, ok := h.session.ReadHave(ctx, module)
 	if !ok {
 		return agent.Have{}, false
@@ -68,7 +76,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
 	if x.Module == agent.NodeModule {
 		// The node's own key holds its facts and the digest of the composition it runs.
 		if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) {
-			h.session.NoteNode(ctx, x.Composed)
+			h.session.NoteNode(ctx, x.Composed, x.Sealed)
 		}
 		return
 	}
@@ -511,6 +519,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			unseal: sealOpener(opts.state),
 			// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
 			held: func(module string) []string {
+				if module == agent.NodeModule {
+					got, _ := session.ReadNodeHave(linked)
+					return got.Sealed
+				}
 				got, ok := session.ReadHave(linked, module)
 				if !ok {
 					return nil

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 	golang.org/x/crypto v0.57.0

go.sum

@@ -1,5 +1,9 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

internal/agent/agent.go

@@ -135,7 +135,7 @@ type Env struct {
 	// Policy is a module's upgrade policy as its manifest derives it (catalogue.DerivedUpgrade); with SelfComposed, a
 	// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
 	// want (ADR 0547 §3). Nil judges no move.
-	Policy func(module string) string
+	Policy func(ctx context.Context, module string) string
 }
 
 // transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
@@ -377,8 +377,8 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
 	}
 }
 
-// writeHave writes a module's have; on a node composing its own, with the sealed values of the composition it names
-// when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else
+// writeHave writes a module's have (the node's own key for NodeModule); on a node composing its own, with the sealed
+// values of the composition it names when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else
 // those the module's last have said at that digest.
 //
 // A writer that names no composition (one written from the agent's running entry before this agent ran anything of the
@@ -394,7 +394,9 @@ func (a *Agent) writeHave(ctx context.Context, h Have) {
 			}
 		}
 	}
-	if a.env.SelfComposed && h.Sealed == nil && h.Module != NodeModule && h.Composed != "" {
+	// The node's own composition too: its core boxes are sealed again by the vault at a restart, and kept in its key the
+	// composer reuses them, so the restart applies nothing again (review of L2, round 3, finding 1).
+	if a.env.SelfComposed && h.Sealed == nil && h.Composed != "" {
 		h.Sealed = a.sealedOf(ctx, h.Module, h.FP, h.Composed)
 	}
 	a.env.Haves.Write(ctx, h)
@@ -1101,7 +1103,7 @@ func short(s string) string {
 // for this composition; the have says phase waiting-yes with the judgement, and the assigner asks a person. A composition
 // equal to what runs is no move. True when the module waits.
 func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp compNow) bool {
-	if module == NodeModule || a.env.Policy == nil || a.env.Policy(module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
+	if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
 		return false
 	}
 	a.mu.Lock()

internal/agent/agent_test.go

@@ -964,7 +964,7 @@ func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
 func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(t *testing.T) {
 	b := newBed(t)
 	b.env.SelfComposed, b.env.Trusted = true, nil
-	b.env.Policy = func(m string) string {
+	b.env.Policy = func(_ context.Context, m string) string {
 		if m == "db" || m == "rec" {
 			return catalogue.PolicyRecord
 		}
@@ -1001,6 +1001,21 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	// The same composition handed again is no move.
 	b.comps.put(ck, craw)
 	time.Sleep(50 * time.Millisecond)
+	// The node's own composition carries its sealed values in its key, as a module's have does, so a restart reuses its
+	// core boxes and applies nothing again (review of L2, round 3, finding 1).
+	box, _ := json.Marshal(map[string]any{"id": "core-box", "type": "file", "path": "/tmp/r6/core-box", "sealed": "core"})
+	own := delivery.Composed{Node: node, Resources: []json.RawMessage{box}}
+	nk, _ := delivery.ComposedNodeKey(node)
+	nraw, _ := delivery.EncodeComposed(own)
+	ndig, _ := delivery.ComposedDigestOf(own)
+	b.comps.put(nk, nraw)
+	nwk, _ := delivery.WantNodeKey(node)
+	nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
+	b.wants.put(nwk, nwraw)
+	eventually(t, "the node's own key does not carry its sealed values", func() bool {
+		h := b.haves.last(NodeModule)
+		return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed[0] == "core"
+	})
 	// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
 	// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
 	// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
@@ -1035,7 +1050,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	// A revision at the same build (a config value changed) is applied.
 	_, craw2, dig2 := unsigned("web", fp, file("web", "x", "2"))
 	b.comps.put(ck, craw2)
-	eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 2 && b.haves.last("web").Composed == dig2 })
+	eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 3 && b.haves.last("web").Composed == dig2 })
 
 	// A record module: its first composition is no move; a new image recreates its container and waits for a yes.
 	container := func(image string) json.RawMessage {
@@ -1046,7 +1061,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
 	b.comps.put(dk, draw)
 	want("db", fp, "")
-	eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 3 })
+	eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
 	fpB := fpOf("b")
 	dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
 	// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the

internal/link/have.go

@@ -47,6 +47,9 @@ type haveState struct {
 	node   *delivery.HaveNode
 	// nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts.
 	nodeComposed string
+	// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
+	// issue 1126): kept in its key so a restart reuses them and applies nothing again.
+	nodeSealed []string
 	// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
 	// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
 	// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
@@ -416,6 +419,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 			var raw []byte
 			facts := *s.node
 			facts.Composed = s.nodeComposed
+			facts.Sealed = s.nodeSealed
 			if raw, err = delivery.EncodeHaveNode(facts); err == nil {
 				want := facts
 				if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
@@ -703,12 +707,14 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
 	q.writeHaves(ctx, bus)
 }
 
-// NoteNode takes the digest of the node's own composition it runs, and writes the node's key where it differs.
-func (q *Session) NoteNode(ctx context.Context, composed string) {
+// NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key
+// (nil for a composition the node did not make itself), and writes the node's key where it differs.
+func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string) {
 	q.init()
 	s := &q.haves
 	s.mu.Lock()
 	s.nodeComposed = composed
+	s.nodeSealed = sealed
 	if s.node == nil {
 		n := delivery.HaveNode{V: delivery.Version}
 		s.node = &n
@@ -720,6 +726,38 @@ func (q *Session) NoteNode(ctx context.Context, composed string) {
 	q.writeHaves(ctx, bus)
 }
 
+// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
+// it last ran is and carries sealed, read after a restart (ADR 0547 §3).
+func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
+	q.init()
+	q.mu.Lock()
+	bus := q.bus
+	q.mu.Unlock()
+	hb, ok := bus.(HaveBus)
+	if !ok || bus == nil {
+		return delivery.HaveNode{}, false
+	}
+	key, err := delivery.HaveNodeKey(q.Membership.Node)
+	if err != nil {
+		return delivery.HaveNode{}, false
+	}
+	read, cancel := context.WithTimeout(ctx, q.timeout())
+	defer cancel()
+	bucket, err := hb.HaveBind(read)
+	if err != nil {
+		return delivery.HaveNode{}, false
+	}
+	value, _, err := bucket.Get(read, key)
+	if err != nil {
+		return delivery.HaveNode{}, false
+	}
+	n, err := delivery.DecodeHaveNode(value)
+	if err != nil {
+		return delivery.HaveNode{}, false
+	}
+	return n, true
+}
+
 // ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
 // be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
 func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {

internal/link/have_test.go

@@ -582,9 +582,12 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
 	if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved {
 		t.Fatalf("removed: %+v", got)
 	}
-	q.NoteNode(context.Background(), dig)
-	if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig {
-		t.Fatalf("the node key: %+v %v", n, err)
+	q.NoteNode(context.Background(), dig, []string{"core"})
+	if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.Sealed) != 1 || n.Sealed[0] != "core" {
+		t.Fatalf("the node key, with its own composition's sealed values: %+v %v", n, err)
+	}
+	if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 {
+		t.Fatalf("the node key read back: %+v %v", n, ok)
 	}
 }
 

internal/selfcompose/selfcompose.go

@@ -453,11 +453,10 @@ func (c *Composer) runningBox(box string) string {
 	}
 	if c.env.Held != nil {
 		c.mu.Lock()
+		// The node's own key among them (agent.NodeModule): its core boxes, as its last run said them.
 		modules := make([]string, 0, len(c.wants))
 		for m := range c.wants {
-			if m != agent.NodeModule {
-				modules = append(modules, m)
-			}
+			modules = append(modules, m)
 		}
 		c.mu.Unlock()
 		sort.Strings(modules)
@@ -514,15 +513,16 @@ func (c *Composer) builtManifest(ctx context.Context, module, fp string) (string
 }
 
 // Policy is a module's upgrade policy as its manifest at the indexer's trunk derives it (catalogue.DerivedUpgrade), for
-// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read.
-func (c *Composer) Policy(module string) string {
+// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read. The manifest is
+// read within ctx, the agent's (review of L2, round 1, finding 5), so a stopping agent stops the read.
+func (c *Composer) Policy(ctx context.Context, module string) string {
 	c.mu.Lock()
 	m, ok := c.modules[module]
 	c.mu.Unlock()
 	if !ok || m.Manifest == "" {
 		return ""
 	}
-	raw, err := reader{c: c}.Manifest(context.Background(), m.Manifest)
+	raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
 	if err != nil {
 		return ""
 	}

internal/selfcompose/selfcompose_test.go

@@ -237,14 +237,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		}
 		return sealed[0]
 	}
+	// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
+	// kept after a restart as a module's is, so the node's own composition does not move either.
 	for _, row := range []struct {
-		name, held string
-		kept       bool
-	}{{"the have holds the rotated value", boxOf(string(rotated.Value)), true}, {"the have holds the value before the rotation", boxOf(before), false}} {
+		name, held, in string
+		kept           bool
+	}{{"the have holds the rotated value", boxOf(string(rotated.Value)), "keeper", true},
+		{"the have holds the value before the rotation", boxOf(before), "keeper", false},
+		{"the node's own key holds the rotated value", boxOf(string(rotated.Value)), agent.NodeModule, true}} {
 		restarted := env
 		restarted.Running = func() []delivery.Composed { return nil }
 		restarted.Held = func(module string) []string {
-			if module == "keeper" {
+			if module == row.in {
 				return []string{row.held}
 			}
 			return nil
@@ -258,7 +262,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 			return nil, fmt.Errorf("not sealed to n1: %s", box)
 		}
 		again := New(restarted)
-		again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}}
+		again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}, agent.NodeModule: {}}
 		rows, err := reader{c: again}.OwnSecrets(ctx, "n1")
 		if err != nil || len(rows) != 1 {
 			t.Fatalf("%s: own secrets %+v, %v", row.name, rows, err)