What runs is the agent's compositions plus the kept declaration's rest: health, filling, take act, directories and the others held out of an apply read it; a restart trusts the node-engine's own haves; the bed tests run off Arch (review round 1)

cmd/mesh-host/filling.go

@@ -27,11 +27,7 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s
 func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
 	return &filling.Engine{
 		Declared: func() (*declaration.Declaration, error) {
-			raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
-			if err != nil {
-				return nil, err
-			}
-			return declaration.Parse(raw)
+			return declaredNow(opts.state, signer)
 		},
 		UIDOf: filling.UIDOf,
 		Lock: func() (func(), error) {

cmd/mesh-host/judge_directories.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 
-	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/store"
@@ -13,16 +12,12 @@ import (
 // judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which
 // the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned.
 func judgeDirectories(statePath string) (string, error) {
-	kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
+	d, err := declaredNow(statePath, nil)
 	if errors.Is(err, store.ErrNothingDeclared) {
 		return "", errors.New("this node keeps no declaration, so it is declared no directory")
 	}
 	if err != nil {
-		return "", err
-	}
-	d, err := declaration.Parse(kept.Declaration)
-	if err != nil {
-		return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
+		return "", fmt.Errorf("what runs on this node does not read: %w", err)
 	}
 	return apply.JudgeDirectories(d), nil
 }

cmd/mesh-host/libraries.go

@@ -251,16 +251,12 @@ func startKeeping(ctx context.Context, say func(string)) {
 // declaration this node keeps, and run beside the applies.
 func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
 	return func(act link.AccessAct) (string, error) {
-		kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
+		d, err := declaredNow(statePath, nil)
 		if errors.Is(err, store.ErrNothingDeclared) {
 			return "", errors.New("this node keeps no declaration, so it is declared no library")
 		}
 		if err != nil {
-			return "", err
-		}
-		d, err := declaration.Parse(kept.Declaration)
-		if err != nil {
-			return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
+			return "", fmt.Errorf("what runs on this node does not read: %w", err)
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), timeout)
 		defer cancel()

cmd/mesh-host/main.go

@@ -30,7 +30,6 @@ import (
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/declaration"
-	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/novox/mesh-host/internal/accounts"
 	"github.com/novox/mesh-host/internal/agent"
 	"github.com/novox/mesh-host/internal/apply"
@@ -1137,6 +1136,7 @@ func runLink(ctx context.Context, opts options) error {
 		Timeout:   opts.timeout,
 	}
 	current := &agentNow{}
+	setRunsHere(runsFromAgent(current))
 	factsOfNow := func(ctx context.Context, ag *agent.Agent) {
 		sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
 	}
@@ -1879,14 +1879,8 @@ func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, sessi
 		return
 	}
 	declared := &declaration.Declaration{}
-	if running := runningOf(ag); len(running) > 0 {
-		if d, err := declarationOf(running...); err == nil {
-			declared = d
-		}
-	} else if raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer); err == nil {
-		if d, err := declaration.Parse(raw); err == nil {
-			declared = d
-		}
+	if d, err := declaredNow(opts.state, signer); err == nil {
+		declared = d
 	}
 	held := map[string]bool{}
 	for _, h := range known.Held {
@@ -1896,13 +1890,6 @@ func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, sessi
 	session.SayFacts(ctx, report)
 }
 
-func runningOf(ag *agent.Agent) []delivery.Composed {
-	if ag == nil {
-		return nil
-	}
-	return ag.Running()
-}
-
 // waitForEnrolment returns this node's identity, waiting for one if it has none.
 //
 // It applies the carried bundle first, if there is one, because that is what a first node does

cmd/mesh-host/nodeagent.go

@@ -228,10 +228,10 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu
 
 // keptByModule is the resources of the declaration the node kept before the switch, by the module they say; the node's
 // own are those that say none (ADR 0539 §9).
-func keptByModule(opts options, signer ed25519.PublicKey) func(module string) ([]json.RawMessage, bool) {
+func keptByModule(opts options, signer ed25519.PublicKey) func() map[string][]json.RawMessage {
 	var once sync.Once
 	var by map[string][]json.RawMessage
-	return func(module string) ([]json.RawMessage, bool) {
+	return func() map[string][]json.RawMessage {
 		once.Do(func() {
 			raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
 			if err != nil {
@@ -256,8 +256,7 @@ func keptByModule(opts options, signer ed25519.PublicKey) func(module string) ([
 				by[name] = append(by[name], r)
 			}
 		})
-		r, ok := by[module]
-		return r, ok
+		return by
 	}
 }
 
@@ -399,3 +398,83 @@ func busyModules() []string {
 	sort.Strings(out)
 	return out
 }
+
+// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
+func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
+	var raw []byte
+	if len(signer) > 0 {
+		var err error
+		if raw, err = store.LoadDeclared(store.DeclaredPath(statePath), signer); err != nil {
+			return nil, err
+		}
+	} else {
+		kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
+		if err != nil {
+			return nil, err
+		}
+		raw = kept.Declaration
+	}
+	return declaration.Parse(raw)
+}
+
+// whatRuns is what runs on this node as the declaration of it (novox/hq ADR 0539 §9): the compositions the agent runs
+// and, for every module the agent does not hold yet, what the kept declaration has for it, with the node's own parts
+// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept
+// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of
+// directories read.
+func whatRuns(everything []delivery.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) {
+	d, err := declarationOf(everything...)
+	if err != nil {
+		return nil, err
+	}
+	if holdsNode {
+		return d, nil
+	}
+	old, err := kept()
+	if err != nil {
+		if len(everything) > 0 {
+			return d, nil
+		}
+		return nil, err
+	}
+	d.Adoption, d.Identities, d.Groups, d.Libraries = old.Adoption, old.Identities, old.Groups, old.Libraries
+	d.Provisioner, d.Builds = old.Provisioner, old.Builds
+	d.Filled = append(d.Filled, old.Filled...)
+	return d, nil
+}
+
+// runsHere says what runs on this node; the running serve sets it to the agent's view, and a command run by hand reads
+// the kept declaration.
+var runsHere struct {
+	mu sync.Mutex
+	fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)
+}
+
+func setRunsHere(fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)) {
+	runsHere.mu.Lock()
+	runsHere.fn = fn
+	runsHere.mu.Unlock()
+}
+
+// declaredNow is the declaration of what runs on this node now.
+func declaredNow(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
+	runsHere.mu.Lock()
+	fn := runsHere.fn
+	runsHere.mu.Unlock()
+	if fn != nil {
+		return fn(statePath, signer)
+	}
+	return keptDeclaration(statePath, signer)
+}
+
+// runsFromAgent is what runs now as the agent in hand sees it.
+func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
+	return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
+		kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) }
+		ag := current.get()
+		if ag == nil {
+			return kept()
+		}
+		return whatRuns(ag.Everything(), ag.HoldsNode(), kept)
+	}
+}

cmd/mesh-host/nodeagent_integration_test.go

@@ -18,9 +18,7 @@ import (
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-host/internal/agent"
-	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/link"
-	"github.com/novox/mesh-host/internal/system"
 )
 
 // The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
@@ -66,15 +64,18 @@ func aServer(t *testing.T) *nats.Conn {
 
 func newItBed(t *testing.T) *itBed {
 	t.Helper()
-	built, err := system.For("arch")
-	if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
-		t.Skip("this machine is not one these tests can apply on")
+	// Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node.
+	bin := t.TempDir()
+	if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
 	}
+	t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
 	was := builtFor
 	builtFor = "arch"
 	t.Cleanup(func() { builtFor = was })
 
 	b := &itBed{t: t, conn: aServer(t), dir: t.TempDir()}
+	var err error
 	if b.js, err = b.conn.JetStream(); err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-host/whatruns_test.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"crypto/ed25519"
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func aFileOf(module, id string) json.RawMessage {
+	raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
+		"content": "x", "module": module})
+	return raw
+}
+
+// What runs is the agent's compositions and the kept declaration's resources for each module the agent does not hold;
+// the node's own parts are the kept declaration's until the agent holds the node's composition (ADR 0539 §9). One module
+// under the agent must not make the rest invisible to the liveness, units, accounts, libraries, filling and the take act.
+func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
+	kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}},
+		Libraries: []declaration.Library{{Library: "media"}}}
+	everything := []delivery.Composed{
+		{Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}},
+		{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's
+	}
+	d, err := whatRuns(everything, false, func() (*declaration.Declaration, error) { return kept, nil })
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(d.Resources) != 2 || len(d.Identities) != 1 || len(d.Libraries) != 1 {
+		t.Fatalf("%d resources, %d identities, %d libraries: the kept modules and the node's kept parts are missing",
+			len(d.Resources), len(d.Identities), len(d.Libraries))
+	}
+
+	// Once the agent holds the node's composition, its parts are the node's.
+	node := delivery.Composed{Node: "n1", Own: delivery.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}}
+	d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(d.Identities) != 1 || d.Identities[0].Name != "mesh-web" || len(d.Libraries) != 0 {
+		t.Fatalf("the node's parts: %+v %+v", d.Identities, d.Libraries)
+	}
+}
+
+// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration.
+func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
+	setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
+		return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil
+	})
+	defer setRunsHere(nil)
+	d, err := declaredNow("/nowhere", nil)
+	if err != nil || d.Provisioner != "from-agent" {
+		t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
+	}
+}

internal/agent/agent.go

@@ -85,9 +85,10 @@ type Env struct {
 	Remove func(ctx context.Context, module string, others []delivery.Composed) Result
 	// Backup backs a module up before a new composition of it is applied; nil backs up nothing.
 	Backup func(ctx context.Context, module string) Backup
-	// Kept is the resources the node last applied for a module from the declaration it kept before the switch (ADR 0539
-	// §9), as JSON, and whether it held the module. Nil when the node kept none.
-	Kept func(module string) ([]json.RawMessage, bool)
+	// Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539
+	// §9), as JSON; the node's own are under NodeModule. Nil when the node kept none. A module the agent does not hold yet
+	// still runs from it.
+	Kept func() map[string][]json.RawMessage
 	// Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A
 	// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
 	// applied. Nil trusts no key, so nothing is applied.
@@ -348,7 +349,8 @@ func (a *Agent) compositionFor(module, fp string) (compNow, bool) {
 	return c, ok
 }
 
-// others are the compositions the node's other modules run now, so what they hold is not taken.
+// others are what the node's other modules run now, so what they hold is not taken: the compositions the agent runs and,
+// for every module the agent does not hold yet, the resources the kept declaration has for it (ADR 0429 §4).
 func (a *Agent) others(except string) []delivery.Composed {
 	var out []delivery.Composed
 	mods := make([]string, 0, len(a.running))
@@ -367,9 +369,44 @@ func (a *Agent) others(except string) []delivery.Composed {
 			out = append(out, c.C)
 		}
 	}
+	if a.env.Kept != nil {
+		kept := a.env.Kept()
+		names := make([]string, 0, len(kept))
+		for m := range kept {
+			if _, held := a.running[m]; !held && m != except {
+				names = append(names, m)
+			}
+		}
+		sort.Strings(names)
+		for _, m := range names {
+			c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]}
+			if m == NodeModule {
+				c.Module = ""
+			}
+			out = append(out, c)
+		}
+	}
 	return out
 }
 
+// Everything is what the node runs: the compositions the agent runs, and the kept declaration's resources for each module
+// the agent does not hold yet. What the node judges its health by, and what the filling, the take act and the judgement
+// of directories read, is built from it.
+func (a *Agent) Everything() []delivery.Composed {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.others("\x00")
+}
+
+// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept
+// declaration's.
+func (a *Agent) HoldsNode() bool {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	h, ok := a.running[NodeModule]
+	return ok && h.Phase != delivery.PhaseRemoved
+}
+
 func (a *Agent) settleOne(ctx context.Context, module string) {
 	a.mu.Lock()
 	w := a.wants[module]
@@ -434,19 +471,24 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 			a.mu.Lock()
 			a.prior[module] = true
 			a.mu.Unlock()
-			if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
-				(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
-				a.mu.Lock()
-				a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
-				a.mu.Unlock()
-				return true
+			if h.Composed != "" {
+				// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
+				// failed apply or a different composition is applied again.
+				if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
+					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
+					a.mu.Lock()
+					a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
+					a.mu.Unlock()
+					return true
+				}
+				return false
 			}
 		}
 	}
 	if a.env.Kept == nil {
 		return false
 	}
-	kept, ok := a.env.Kept(module)
+	kept, ok := a.env.Kept()[module]
 	if ok {
 		a.mu.Lock()
 		a.prior[module] = true

internal/agent/agent_test.go

@@ -110,15 +110,17 @@ type bed struct {
 	failWith string
 	backup   func(module string) Backup
 	kept     map[string][]json.RawMessage
+	others   [][]delivery.Composed
 	env      Env
 }
 
 func newBed(t *testing.T) *bed {
 	b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
 	b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
-		Apply: func(_ context.Context, c delivery.Composed, _ []delivery.Composed) Result {
+		Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
 			b.mu.Lock()
 			defer b.mu.Unlock()
+			b.others = append(b.others, others)
 			name := c.Module
 			if name == "" {
 				name = NodeModule
@@ -144,10 +146,7 @@ func newBed(t *testing.T) *bed {
 			}
 			return Backup{Done: true}
 		},
-		Kept: func(m string) ([]json.RawMessage, bool) {
-			r, ok := b.kept[m]
-			return r, ok
-		},
+		Kept: func() map[string][]json.RawMessage { return b.kept },
 		Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} },
 		Say:     func(s string) { t.Log(s) },
 	}
@@ -556,3 +555,88 @@ func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) {
 		t.Fatalf("applied with no trusted key: %v", got)
 	}
 }
+
+// What a module's apply must not take is every other module that runs, those the agent holds and those still running from
+// the kept declaration (ADR 0429 §4): the resources of the kept declaration reach Apply as the others.
+func TestApplyingOneModuleHoldsOutTheModulesStillRunningFromTheKeptDeclaration(t *testing.T) {
+	b := newBed(t)
+	b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, NodeModule: {file("node", "n", "n")}}
+	stop := b.run()
+	defer stop()
+	fp := fpOf("a")
+	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+	b.comps.put(ck, craw)
+	wk, wraw := wantFor(t, "web", fp, dig, "")
+	b.wants.put(wk, wraw)
+	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var names []string
+	for _, c := range b.others[0] {
+		names = append(names, c.Module)
+	}
+	if len(names) != 2 || !strings.Contains(strings.Join(names, ","), "db") {
+		t.Fatalf("the others of web were %v: db and the node's own run from the kept declaration", names)
+	}
+}
+
+// Everything is what runs: the agent's compositions and the kept resources of each module it does not hold.
+func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) {
+	b := newBed(t)
+	b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, "web": {file("web", "x", "old")}}
+	a := New(b.env)
+	if got := a.Everything(); len(got) != 2 {
+		t.Fatalf("before the agent holds anything: %d compositions, want the 2 kept", len(got))
+	}
+	a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: "sha256:" + strings.Repeat("1", 64), Phase: delivery.PhaseRunning}
+	ck, craw, dig := composition(t, "web", fpOf("a"), false, file("web", "x", "new"))
+	a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: dig, Phase: delivery.PhaseRunning}
+	a.takeComposition(Update{Key: ck, Value: craw, Revision: 1})
+	got := a.Everything()
+	if len(got) != 2 {
+		t.Fatalf("%d compositions", len(got))
+	}
+	for _, c := range got {
+		if c.Module == "web" && !strings.Contains(string(c.Resources[0]), "new") {
+			t.Fatal("web is the kept declaration's, not the composition the agent runs")
+		}
+	}
+	if a.HoldsNode() {
+		t.Fatal("the agent holds no node composition")
+	}
+}
+
+// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
+// node-engine wrote its haves, they are what it knows (the kept declaration is only for the first start after the switch).
+func TestAFailedApplyIsAppliedAgainAfterARestartWhateverIsKept(t *testing.T) {
+	b := newBed(t)
+	fp := fpOf("a")
+	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+	wk, wraw := wantFor(t, "web", fp, dig, "")
+	b.comps.before(ck, craw)
+	b.wants.before(wk, wraw)
+	b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
+	b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed)}}
+	stop := b.run()
+	defer stop()
+	eventually(t, "a failed apply was marked running from the kept declaration", func() bool { return len(b.appliedList()) == 1 })
+}
+
+// A have written before the switch carries no composition: then the kept declaration says what runs.
+func TestAHaveWithNoCompositionDefersToTheKeptDeclaration(t *testing.T) {
+	b := newBed(t)
+	fp := fpOf("a")
+	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+	wk, wraw := wantFor(t, "web", fp, dig, "")
+	b.comps.before(ck, craw)
+	b.wants.before(wk, wraw)
+	b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
+	b.haves.stored = map[string]Have{"web": {Module: "web", Phase: "running"}}
+	stop := b.run()
+	defer stop()
+	eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
+	time.Sleep(50 * time.Millisecond)
+	if got := b.appliedList(); len(got) != 0 {
+		t.Fatalf("applied %v", got)
+	}
+}