Keep the packages this node's compositions declare in the engine's own key, and answer them on the node seat (ADR 0565 clauses 6 and 15, issue 1953)

pacman still reads the controller's mirror because no engine wrote its key: the engine
now keeps each handed composition's package resources and, after a pass that handed
something new, writes mesh-host_packages.<node> only when the value differs, owed after
the growing bound when refused. The manifest declares the state so the grants part
creates it. The verb packages answers the key beside the mirror's key for the node and
their difference, and with write writes the key again from what was composed. H8's
integration test proves the write under the grant the grants part composes.

Needs mesh-sdk feat/node-packages (0ef7c0c) for DeclaredPackagesOf and the engine's read of the
mirror (go.mod pin left for the lead: the SDK commit is not pushed).

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL

internal/selfcompose/data_test.go

@@ -177,6 +177,56 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) {
 			}
 		})
 	}
+	// ADR 0565 §6 (1953): the node's key of the engine's state packages is written after a pass that handed something new
+	// when the packages its compositions declare differ, and only then: a composition adding a package writes it once,
+	// one that changes no package writes nothing; a write refused is owed after the growing bound; the verb's write reads
+	// the key again, so a key lost on the bus is written back. Another node's key is never touched.
+	pkgStore := &fakeData{kv: map[string][]byte{"n2": []byte(`{"node":"n2","packages":[]}`)}}
+	pc := New(Env{Node: "n1", Packages: pkgStore})
+	web := func(pkgs ...string) delivery.Composed {
+		comp := delivery.Composed{Node: "n1", Module: "web"}
+		for _, p := range pkgs {
+			comp.Resources = append(comp.Resources, []byte(`{"id":"`+p+`","type":"package","package":"`+p+`"}`))
+		}
+		comp.Resources = append(comp.Resources, []byte(`{"id":"conf","type":"file","path":"/etc/web.conf"}`))
+		return packagesOf(comp)
+	}
+	for _, step := range []struct {
+		name  string
+		comp  delivery.Composed
+		fail  error
+		puts  int
+		owed  bool
+		holds string
+	}{
+		{name: "the run's first pass writes the key", comp: web("curl"), puts: 1,
+			holds: `{"node":"n1","packages":[{"package":"curl","modules":["web"],"absent":false}]}`},
+		{name: "a pass that changes no package writes nothing", comp: web("curl"), puts: 1},
+		{name: "a refused write is owed after the bound", comp: web("curl", "jq"), fail: errors.New("no grant yet"), puts: 1, owed: true},
+		{name: "a composition adding a package writes the key once", comp: web("curl", "jq"), puts: 2,
+			holds: `{"node":"n1","packages":[{"package":"curl","modules":["web"],"absent":false},{"package":"jq","modules":["web"],"absent":false}]}`},
+	} {
+		pkgStore.mu.Lock()
+		pkgStore.fail = step.fail
+		pkgStore.mu.Unlock()
+		pc.paid()
+		pc.mu.Lock()
+		pc.pkgs, pc.pkgsComposed, pc.pkgsDue = map[string]delivery.Composed{webKey: step.comp}, true, true
+		pc.mu.Unlock()
+		pc.publishPackages(context.Background())
+		if len(pkgStore.puts) != step.puts || pc.owed.Load() != step.owed ||
+			(step.holds != "" && string(pkgStore.kv["n1"]) != step.holds) || string(pkgStore.kv["n2"]) != `{"node":"n2","packages":[]}` {
+			t.Errorf("%s: %d writes, owed %v, n1 holds %s", step.name, len(pkgStore.puts), pc.owed.Load(), pkgStore.kv["n1"])
+		}
+	}
+	delete(pkgStore.kv, "n1")
+	if wrote, err := pc.WritePackages(context.Background()); !wrote || err != nil || pkgStore.kv["n1"] == nil {
+		t.Errorf("the verb's write of a lost key wrote %v (%v)", wrote, err)
+	}
+	if wrote, _ := pc.WritePackages(context.Background()); wrote {
+		t.Error("the verb's write of a key holding what was composed wrote it again")
+	}
+
 	// The items are worked out from what the pass read: a done build is read over the bus once, not again for the data
 	// items after the pass (a module in a running build is read again, since it may land).
 	b := newFakeBus()

internal/selfcompose/packages.go

@@ -0,0 +1,152 @@
+package selfcompose
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// **The packages this node's compositions declare, kept by its engine** (novox/hq ADR 0565 §6, §15; issue 1953). The
+// composer keeps the package resources of each composition it handed, and after a pass that handed something new (or
+// the run's first) it works out the node's key of the engine's state packages from them (delivery.DeclaredPackagesOf,
+// the controller mirror's layout and order) and writes it only when the value differs from what the key holds (ADR 0289
+// §1): no timer, and no composing of another node. A module held keeps the packages of what was handed of it, as it keeps
+// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it beside the
+// controller's mirror until the lease write, and with write writes it again (ADR 0565 §15).
+
+// PackagesStore is this node's key of the engine's state packages (link.EnginePackages).
+type PackagesStore interface {
+	// Get is the key's value, nil when it has none.
+	Get(ctx context.Context, key string) ([]byte, error)
+	Put(ctx context.Context, key string, value []byte) error
+}
+
+// packagesOf is a composition's package resources alone, with its module: what the node's key is worked out from.
+func packagesOf(comp delivery.Composed) delivery.Composed {
+	out := delivery.Composed{Node: comp.Node, Module: comp.Module}
+	for _, raw := range comp.Resources {
+		var head struct {
+			Type string `json:"type"`
+		}
+		if json.Unmarshal(raw, &head) == nil && head.Type == "package" {
+			out.Resources = append(out.Resources, raw)
+		}
+	}
+	return out
+}
+
+// DeclaredPackages is the node's key as this composer would write it now, from what it handed.
+func (c *Composer) DeclaredPackages() (delivery.DeclaredPackages, error) {
+	c.mu.Lock()
+	keys := make([]string, 0, len(c.pkgs))
+	for k := range c.pkgs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	comps := make([]delivery.Composed, 0, len(keys))
+	for _, k := range keys {
+		comps = append(comps, c.pkgs[k])
+	}
+	composed := c.pkgsComposed
+	c.mu.Unlock()
+	if !composed {
+		return delivery.DeclaredPackages{}, errors.New("this node's composer has handed no composition yet in this run, so it " +
+			"declares no packages yet")
+	}
+	return delivery.DeclaredPackagesOf(c.env.Node, comps)
+}
+
+// publishPackages writes the node's key of the state packages when a pass asked it (pkgsDue) and the value differs.
+func (c *Composer) publishPackages(ctx context.Context) {
+	c.mu.Lock()
+	due := c.pkgsDue
+	c.pkgsDue = false
+	c.mu.Unlock()
+	if !due || c.env.Packages == nil {
+		return
+	}
+	wrote, err := c.writePackages(ctx, false)
+	if err != nil {
+		c.sayPackages("this node's packages are not kept in " + enginePackagesBucket + "; " + c.owePackages() + ": " + err.Error())
+		return
+	}
+	c.mu.Lock()
+	c.pkgsTries = 0
+	c.mu.Unlock()
+	c.sayPackages("")
+	if wrote {
+		c.env.Say("this node's packages: its key of " + enginePackagesBucket + " written")
+	}
+}
+
+// enginePackagesBucket is the engine's state packages as the bus names it, for what the composer says.
+const enginePackagesBucket = "mesh-host_" + delivery.EnginePackagesState
+
+// WritePackages writes the node's key again from what the composer handed, when it differs from what the bus holds
+// now (the node seat's verb packages with write, ADR 0565 §15): the key is read again rather than trusted, so a key lost
+// or changed on the bus is written back. It answers whether it wrote.
+func (c *Composer) WritePackages(ctx context.Context) (bool, error) {
+	if c.env.Packages == nil {
+		return false, errors.New("this engine keeps no packages key")
+	}
+	return c.writePackages(ctx, true)
+}
+
+// writePackages writes the key when the value worked out differs from what it holds: read once per run (pkgsKept), or
+// again when fresh says so.
+func (c *Composer) writePackages(ctx context.Context, fresh bool) (bool, error) {
+	v, err := c.DeclaredPackages()
+	if err != nil {
+		return false, err
+	}
+	raw, err := json.Marshal(v)
+	if err != nil {
+		return false, err
+	}
+	c.pkgsWrite.Lock()
+	defer c.pkgsWrite.Unlock()
+	if fresh || !c.pkgsRead {
+		kept, err := c.env.Packages.Get(ctx, c.env.Node)
+		if err != nil {
+			return false, fmt.Errorf("%s cannot be read: %w", c.env.Node, err)
+		}
+		c.pkgsKept, c.pkgsRead = kept, true
+	}
+	if bytes.Equal(c.pkgsKept, raw) {
+		return false, nil
+	}
+	if err := c.env.Packages.Put(ctx, c.env.Node, raw); err != nil {
+		return false, err
+	}
+	c.pkgsKept = raw
+	return true, nil
+}
+
+// owePackages keeps the packages due and has the loop make a pass again after the growing bound, as oweData does for the
+// data items: the first write after a delivery may be refused while the state or its grant is not on the bus yet.
+func (c *Composer) owePackages() string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.pkgsDue = true
+	c.wait = LaterWait(c.pkgsTries)
+	c.pkgsTries++
+	c.owed.Store(true)
+	return fmt.Sprintf("tried again at %s (try %d)", time.Now().Add(c.wait).Format("15:04:05"), c.pkgsTries)
+}
+
+// sayPackages says a line about the packages once until it changes; "" clears it.
+func (c *Composer) sayPackages(line string) {
+	c.mu.Lock()
+	same := c.pkgsSaid == line
+	c.pkgsSaid = line
+	c.mu.Unlock()
+	if !same && line != "" {
+		c.env.Say(line)
+	}
+}

internal/selfcompose/selfcompose.go

@@ -90,6 +90,9 @@ type Env struct {
 	// Data is this node's keys of the engine's state data (novox/hq ADR 0577 clause 1), where each placed module's data
 	// items are kept after a pass that composed something new. Nil keeps none.
 	Data DataStore
+	// Packages is this node's key of the engine's state packages (novox/hq ADR 0565 §6), written after a pass that
+	// handed something new when the packages its compositions declare differ. Nil keeps none.
+	Packages PackagesStore
 }
 
 // VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
@@ -161,6 +164,17 @@ type Composer struct {
 	dataSaid  string
 	dataScope frombus.Scope
 	dataKept  map[string][]byte
+	// pkgs is the package resources of each composition handed, by its composed key, and pkgsComposed whether this run
+	// handed any (ADR 0565 §6); pkgsDue, pkgsTries and pkgsSaid are as the data items' are. Under mu. pkgsKept is what
+	// the node's key of the state packages held when last read or written (pkgsRead), under pkgsWrite.
+	pkgs         map[string]delivery.Composed
+	pkgsComposed bool
+	pkgsDue      bool
+	pkgsTries    int
+	pkgsSaid     string
+	pkgsWrite    sync.Mutex
+	pkgsKept     []byte
+	pkgsRead     bool
 }
 
 // Pass is how one pass went, as the node-engine's verb compose answers it.
@@ -241,7 +255,7 @@ func New(env Env) *Composer {
 	return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, placementKeys: delivery.NewPlacementKeys(),
 		records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
 		manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]delivery.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{},
-		asked: make(chan chan Pass, 8)}
+		pkgs: map[string]delivery.Composed{}, asked: make(chan chan Pass, 8)}
 }
 
 // ComposeNow asks the composer for a pass now (the node-engine's verb compose, issue 774): the growing bound after a seat
@@ -453,6 +467,7 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
 				}
 			}
 			c.publishData(ctx)
+			c.publishPackages(ctx)
 			if c.owed.Load() {
 				wait, _ := c.waitNow()
 				c.env.Say(fmt.Sprintf("the node composes again in %s: a read on the bus did not answer", wait))
@@ -903,6 +918,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 		}
 		c.rev++
 		c.handed[key] = dig
+		c.pkgs[key] = packagesOf(comp)
 		out = append(out, agent.Update{Key: key, Value: raw, Revision: c.rev})
 	}
 	for key := range c.handed {
@@ -914,12 +930,16 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 		if !now[key] {
 			c.rev++
 			delete(c.handed, key)
+			delete(c.pkgs, key)
 			out = append(out, agent.Update{Key: key, Deleted: true, Revision: c.rev})
 		}
 	}
 	pass.Handed = len(out)
 	// The data items are kept again after a pass that handed something new, and after the run's first (ADR 0577 clause 1).
 	c.dataDue = c.dataDue || len(out) > 0 || c.dataKept == nil
+	// And the node's packages after the same passes (ADR 0565 §6): written only when they differ.
+	c.pkgsDue = c.pkgsDue || len(out) > 0 || !c.pkgsComposed
+	c.pkgsComposed = true
 	return out
 }
 

internal/selfcompose/selfcompose_test.go

@@ -393,7 +393,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		t.Fatal(err)
 	}
 	proxy := `{"module":"proxy","version":"1","capabilities":["systemd"],"provides":[{"name":"route","scope":"mesh"},{"name":"smtp","scope":"mesh"}],` +
-		`"resources":[{"id":"service","type":"service","unit":"proxy.service","state":"running","boot":"enabled"}]}`
+		`"resources":[{"id":"service","type":"service","unit":"proxy.service","state":"running","boot":"enabled"},` +
+		`{"id":"pkg","type":"package","package":"haproxy"}]}`
 	c3, told := fresh(env.Get)
 	for m, raw := range map[string]string{"de-spiegel": string(spiegel), "proxy": proxy} {
 		dig := delivery.ManifestDigest([]byte(raw))
@@ -430,6 +431,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	if u, ok := handed3[spiegelKey]; ok {
 		t.Errorf("held de-spiegel is handed %+v, want what was handed of it kept", u)
 	}
+	// ADR 0565 §6 (1953): the pass keeps the packages each handed composition declares, with their module, for the node's
+	// key of the engine's state packages.
+	if d, err := c3.DeclaredPackages(); err != nil || len(d.Packages) != 1 || d.Packages[0].Package != "haproxy" ||
+		strings.Join(d.Packages[0].Modules, ",") != "proxy" || d.Node != "n1" {
+		t.Errorf("the node's packages after the pass are %+v (%v)", d, err)
+	}
 	if len(*told) != 1 || !strings.HasPrefix((*told)[0], "held de-spiegel: ") || !strings.Contains((*told)[0], "${config:praktijk-name}") ||
 		!strings.Contains((*told)[0], "without what it contributes to route") || strings.Contains((*told)[0], "held proxy") ||
 		strings.Contains((*told)[0], "held lemurs") {