cmd/mesh-controller/composed_keys.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -114,26 +116,64 @@ func inList(list []string, s string) bool {
 // doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
 var doneKeysMu sync.Map
 
-// takeDoneKey is a woken module's take-in on the live stores and buckets, said.
+// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
+// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
+// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
+// yet is no failure: its next write wakes this again.
 func takeDoneKey(ctx context.Context, open *stores, module string) {
 	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
 	mu.(*sync.Mutex).Lock()
 	defer mu.(*sync.Mutex).Unlock()
+	failed := false
+	for {
+		err := takeDoneKeyOnce(ctx, open, module)
+		o := doneKeyObservation(module)
+		if err == nil || errors.Is(err, errKeyWaits) {
+			if keeper := conditionsFrom; keeper != nil && failed {
+				_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
+			}
+			return
+		}
+		fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry)
+		if keeper := conditionsFrom; keeper != nil && !failed {
+			o.Summary += err.Error()
+			if _, oerr := keeper.Observe(ctx, o); oerr == nil {
+				failed = true
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(doneKeyRetry):
+		}
+	}
+}
+
+// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
+var doneKeyRetry = 30 * time.Second
+
+// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
+// they are, no want can name it.
+const kindNotComposed = "fingerprint-not-composed"
+
+func doneKeyObservation(module string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
+		Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
+		Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
+}
+
+// takeDoneKeyOnce is one take-in of a module's done key.
+func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
 	keys, closeKeys, err := openBuilderKeys()
 	if err != nil {
-		fmt.Printf("done keys: %s: the builder's keys cannot be read: %v\n", module, err)
-		return
+		return fmt.Errorf("the builder's keys cannot be read: %w", err)
 	}
 	defer closeKeys()
 	said, err := liveDoneKeySteps(open, keys).take(ctx, module)
-	switch {
-	case errors.Is(err, errKeyWaits):
-		// Said by nobody: the key's next write wakes this again.
-	case err != nil:
-		fmt.Printf("done keys: %s: %v\n", module, err)
-	case said != "":
+	if err == nil && said != "" {
 		fmt.Printf("done keys: %s\n", said)
 	}
+	return err
 }
 
 // liveDoneKeySteps are the steps on the controller's own stores and the bus.
@@ -197,8 +237,7 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
 			if err != nil {
 				return err
 			}
-			grantsIssued(ctx, inv, js, "done keys: ")
-			return nil
+			return grantsIssued(ctx, inv, js, "done keys: ")
 		},
 	}
 }

cmd/mesh-controller/composed_keys_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"encoding/json"
 	"strings"
 	"testing"
@@ -114,3 +115,15 @@ func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testin
 		}
 	}
 }
+
+// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left,
+// nothing else would write it, and no want could name the fingerprint.
+func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) {
+	fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
+	var acts []string
+	s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox")
+	s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") }
+	if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") {
+		t.Errorf("a grant not written: %v", err)
+	}
+}

cmd/mesh-controller/grants.go

@@ -126,28 +126,25 @@ func withVersion(g sdkdelivery.ControllerGrant) sdkdelivery.ControllerGrant {
 	return g
 }
 
-// grantsIssued writes the grants the bus's user list just sent to its machine issued (ADR 0528 clause 2). Said, never
-// fatal: a grant not written now is written at the next send of the list, and the assigner's wants wait for it.
-func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) {
+// grantsIssued writes the grants issued for every module whose credentials all exist at the fingerprint it is
+// registered at (ADR 0528 clause 2, ADR 0539 clause 7). Its error is the caller's: with no send left, nothing writes a
+// missed grant later but the caller's retry (the done key's take-in, composed_keys.go), and the assigner's wants wait.
+func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) error {
 	kv, err := js.KeyValue(ctx, broker.GrantsBucket)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: the bucket %s: %v\n", indent, broker.GrantsBucket, err)
-		return
+		return fmt.Errorf("the grants issued are not written: the bucket %s: %w", broker.GrantsBucket, err)
 	}
 	records, err := inv.BusRecords(ctx)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	principals, err := broker.Users(records)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	kept, err := inv.BusUsers(ctx)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	hashes := make(map[string]string, len(kept))
 	for name, u := range kept {
@@ -155,14 +152,14 @@ func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.Je
 	}
 	current, err := currentFingerprints(ctx, inv)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	wrote, err := keepGrants(ctx, kv, grantsFrom(principals, hashes, current))
 	if wrote > 0 {
 		fmt.Printf("%s%d grant(s) issued written for the assigner\n", indent, wrote)
 	}
 	if err != nil {
-		fmt.Printf("%ssome grants issued are not written: %v\n", indent, err)
+		return fmt.Errorf("some grants issued are not written: %w", err)
 	}
+	return nil
 }

internal/broker/composed.go

@@ -120,7 +120,47 @@ func NodeAgentAsks(node string) ([]string, error) {
 
 // NodeAgentHears are the events a node's node-engine subscribes while it applies a want: the backup holder's word that a
 // backup is taken (`backed-up`), from whichever module holds node-backup on its machine (restic today).
-var NodeAgentHears = []string{"mesh.mod.*.event.backed-up"}
+var NodeAgentHears = []string{"mesh.mod." + BackupHolderModule + ".event.backed-up"}
+
+// BackupHolderModule is the module that holds node-backup on every node, restic: the one whose word a node-engine hears.
+// A second holder of the seat is a change of this grant.
+const BackupHolderModule = "restic"
+
+// administers says a person may invoke every tool (`*`): the operator.
+func administers(invokes []string) bool {
+	for _, i := range invokes {
+		if i == "*" {
+			return true
+		}
+	}
+	return false
+}
+
+// HandPathWrites is what the operator's account publishes for release 6's hand path (ADR 0539 clauses 2 and 10): a want
+// of any node, written and read back, in the assigner's bucket; each named node's compositions; the digests; and each
+// node's object bucket for a file over 64 KiB, whose put reads an object's metadata and purges a replaced one's chunks.
+func HandPathWrites(nodes []string) ([]string, error) {
+	out := append([]string{"$KV." + delivery.AssignerWantsBucket + ".want.>"}, readGrants(delivery.AssignerWantsBucket, "want.>")...)
+	out = append(out, "$KV."+delivery.ControllerComposedBucket+".>")
+	out = append(out, readGrants(delivery.ControllerComposedBucket, "")...)
+	for _, node := range nodes {
+		w, err := ComposedWrites(node)
+		if err != nil {
+			return nil, err
+		}
+		bucket, _ := delivery.ComposedBucketFor(node)
+		out = append(out, w)
+		out = append(out, readGrants(bucket, "")...)
+		files, err := DeclarationFilesWrites(node)
+		if err != nil {
+			return nil, err
+		}
+		stream := "OBJ_" + DeclarationFilesBucket + node
+		out = append(out, files, "$JS.API.STREAM.INFO."+stream, "$JS.API.DIRECT.GET."+stream+".>",
+			"$JS.API.STREAM.MSG.GET."+stream, "$JS.API.STREAM.PURGE."+stream)
+	}
+	return out, nil
+}
 
 // DeclarationsStream is the stream of each node's last declaration, `mesh.node.<node>.declare` (issue 107). Release 6
 // sends none (ADR 0539 clauses 5 and 11): the stream is kept, with each node's consumer, only so a node away during the

internal/broker/composed_live_test.go

@@ -0,0 +1,145 @@
+//go:build integration
+
+package broker
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// 1945 (the review of A1, H1): on a bus running the user list this controller composes, the operator's account (a
+// person who may invoke every tool) walks release 6's hand path end to end: it writes a node's want, that node's
+// composition and its digest, and stores a file over 64 KiB in the node's object bucket and replaces it; a person with
+// fewer tools may do none of it.
+func TestNatsTheOperatorWritesTheHandPathOnTheComposedUserList(t *testing.T) {
+	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	users, err := Users(Records{Nodes: []string{"ace"},
+		People: map[string][]string{"jochen": {"*"}, "guest": {"mesh-controller.status"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range users {
+		users[i].PasswordHash = string(hash)
+	}
+	accounts, err := ComposeAccounts(users)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := filepath.Join(t.TempDir(), "bus.conf")
+	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	opts, err := server.ProcessConfigFile(conf)
+	if err != nil {
+		t.Fatalf("the composed accounts do not parse: %v", err)
+	}
+	opts.NoLog, opts.NoSigs = true, true
+	s, err := server.NewServer(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go s.Start()
+	if !s.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	t.Cleanup(s.Shutdown)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	connect := func(user string) jetstream.JetStream {
+		t.Helper()
+		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user))
+		if err != nil {
+			t.Fatalf("%s cannot connect: %v", user, err)
+		}
+		t.Cleanup(nc.Close)
+		js, err := jetstream.New(nc)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return js
+	}
+
+	// The buckets as the controller and the assigner make them.
+	cjs := connect(ControllerName)
+	if err := EnsureDeclarationFiles(ctx, cjs, "ace"); err != nil {
+		t.Fatal(err)
+	}
+	if err := EnsureComposed(ctx, cjs); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := cjs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "composed-ace", History: 5}); err != nil {
+		t.Fatal(err)
+	}
+	// The assigner's bucket, made here by the controller's account, which may create any stream.
+	if _, err := cjs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-delivery-assigner_wants", History: 20}); err != nil {
+		t.Fatal(err)
+	}
+
+	short := func(c context.Context) (context.Context, context.CancelFunc) {
+		return context.WithTimeout(c, 5*time.Second)
+	}
+	for _, who := range []struct {
+		user string
+		may  bool
+	}{{"person.jochen", true}, {"person.guest", false}} {
+		js := connect(who.user)
+		try := func(what string, act func(context.Context) error) {
+			t.Helper()
+			c, stop := short(ctx)
+			defer stop()
+			err := act(c)
+			if (err == nil) != who.may {
+				t.Errorf("%s: %s: %v (may: %v)", who.user, what, err, who.may)
+			}
+		}
+		try("write ace's want", func(c context.Context) error {
+			kv, err := js.KeyValue(c, "mesh-delivery-assigner_wants")
+			if err != nil {
+				return err
+			}
+			_, err = kv.Put(c, "want.ace.mesh-controller", []byte(`{"v":1}`))
+			return err
+		})
+		try("write ace's composition", func(c context.Context) error {
+			kv, err := js.KeyValue(c, "composed-ace")
+			if err != nil {
+				return err
+			}
+			_, err = kv.Put(c, "ace.mesh-controller.aaaa", []byte(`{"v":1}`))
+			return err
+		})
+		try("write its digest", func(c context.Context) error {
+			kv, err := js.KeyValue(c, delivery.ControllerComposedBucket)
+			if err != nil {
+				return err
+			}
+			_, err = kv.Put(c, "ace.mesh-controller.aaaa", []byte(`{"v":1}`))
+			return err
+		})
+		try("store and replace a file over 64 KiB", func(c context.Context) error {
+			store, err := js.ObjectStore(c, "declaration-files-ace")
+			if err != nil {
+				return err
+			}
+			body := bytes.Repeat([]byte("x"), 200<<10)
+			if _, err := store.PutBytes(c, "f00d", body); err != nil {
+				return err
+			}
+			_, err = store.PutBytes(c, "f00d", append(body, 'y'))
+			return err
+		})
+	}
+}

internal/broker/composed_test.go

@@ -65,7 +65,7 @@ func TestANodeAgentAsksItsOwnBackupHolder(t *testing.T) {
 			t.Errorf("a node-engine of ace may not publish %s", want)
 		}
 	}
-	if !strings.Contains(strings.Join(p.Subscribe, " "), "mesh.mod.*.event.backed-up") {
+	if !strings.Contains(strings.Join(p.Subscribe, " "), "mesh.mod.restic.event.backed-up") {
 		t.Errorf("a node-engine of ace does not hear a backup taken: %v", p.Subscribe)
 	}
 	if strings.Contains(all, "node-backup.tool.now.*") || strings.Contains(all, "KV_composed-g14") {
@@ -106,3 +106,44 @@ func TestARaiseAfterTheDeclarationsAssertsNoneOfThem(t *testing.T) {
 		t.Errorf("after the retirement %d streams are asserted, before %d", len(after.streams), len(before.streams))
 	}
 }
+
+// 1945 (the review of A1, (b) and H1): the operator's account, a person who may invoke every tool, writes release 6's
+// hand path: a want of any node, each node's compositions, the digests, and a node's object bucket with what replacing
+// an object needs; a person with fewer tools writes none of it.
+func TestTheOperatorWritesTheHandPath(t *testing.T) {
+	users, err := Users(Records{Nodes: []string{"ace", "novox"}, People: map[string][]string{"jochen": {"*"}, "guest": {"mesh-controller.status"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, u := range users {
+		if u.Kind != KindPerson {
+			continue
+		}
+		perms, err := PermissionsFor(u)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, s := range []string{"$KV.mesh-delivery-assigner_wants.want.>", "$KV.composed-ace.>", "$KV.composed-novox.>",
+			"$KV.mesh-controller_composed.>", "$O.declaration-files-ace.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-ace.>",
+			"$JS.API.STREAM.MSG.GET.OBJ_declaration-files-ace", "$JS.API.STREAM.PURGE.OBJ_declaration-files-ace"} {
+			found := false
+			for _, p := range perms.Publish {
+				found = found || p == s
+			}
+			if found != (u.Module == "jochen") {
+				t.Errorf("%s may publish %s: %v", u.Module, s, found)
+			}
+		}
+	}
+	c, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	for _, p := range c.Publish {
+		found = found || p == "mesh.seat.mesh-controller.event.bucket-deleted"
+	}
+	if !found {
+		t.Errorf("the controller may not say bucket-deleted")
+	}
+}

internal/broker/genesis_template_test.go

@@ -49,7 +49,9 @@ func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T)
 	// builder (issue 1152): the controller derives none of them (R4g, ADR 0525 §10), the first push takes them away.
 	carriedPub = slices.DeleteFunc(slices.Clone(carriedPub), func(s string) bool {
 		return strings.HasPrefix(s, "mesh.seat.node-build-agent.") || strings.HasPrefix(s, "mesh.seat.mesh-build-machine.") ||
-			strings.HasPrefix(s, "$KV.SEAT_NODE_BUILD_AGENT_") || strings.HasPrefix(s, "$KV.SEAT_MESH_BUILD_MACHINE_")
+			strings.HasPrefix(s, "$KV.SEAT_NODE_BUILD_AGENT_") || strings.HasPrefix(s, "$KV.SEAT_MESH_BUILD_MACHINE_") ||
+			// And the plans' events release 6 retired (novox/hq ADR 0539), until mesh-host's genesis template drops them.
+			s == "mesh.seat.mesh-controller.event.plan-moved" || s == "mesh.seat.mesh-controller.event.sent"
 	})
 	if diff := missing(carriedPub, want.Publish); len(diff) > 0 {
 		t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff)

internal/broker/log_test.go

@@ -211,7 +211,10 @@ func TestNoPathDeletesALogOrABucket(t *testing.T) {
 			for _, lit := range stringsIn(f) {
 				if strings.Contains(lit, "$JS.API.STREAM.DELETE") || strings.Contains(lit, "$JS.API.STREAM.PURGE") {
 					// Only the writers table names it, as a subject no one but the controller may publish.
-					if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") {
+					// And the operator's release 6 hand path, whose object store purges a replaced object's chunks in a node's
+					// object bucket (novox/hq ADR 0539 clause 10): a grant, never a call of this repository.
+					if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") &&
+						!strings.HasSuffix(filepath.ToSlash(path), "internal/broker/composed.go") {
 						found = append(found, path+": "+lit)
 					}
 				}

internal/broker/nats.go

@@ -422,6 +422,11 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
 			"first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)",
 	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".event.bucket-deleted",
+		Reason: "refused: nothing is deleted at genesis; the controller composes its own permissions into the bus module's " +
+			"composition before a later start's deletions (novox/hq ADR 0539 clause 13)",
+	},
 	{
 		Subject: "$KV." + delivery.ControllerComposedBucket + ".>",
 		Reason: "carried only at genesis: the controller composes its own permissions into the bus module's composition, " +
@@ -774,6 +779,15 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// And may ask what answers (novox/hq ADR 0197): a question every service answers about
 		// itself, its replies to the asker's own inbox.
 		pub = append(pub, discovering()...)
+		// And, for an administrator (`*`, the operator), release 6's hand path (ADR 0539 clauses 2 and 10): a want written
+		// at the terminal, and a composition made by hand with mesh-compose.
+		if administers(p.Invokes) {
+			hand, err := HandPathWrites(p.FilesFor)
+			if err != nil {
+				return Permissions{}, err
+			}
+			pub = append(pub, hand...)
+		}
 
 	case KindEnrolment:
 		// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear

internal/broker/nats_test.go

@@ -241,7 +241,12 @@ func TestAnAdministratorMayAskAnyTool(t *testing.T) {
 func TestAPersonReachesNothingButTools(t *testing.T) {
 	perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
 		Invokes: []string{"*"}, PasswordHash: "x"})
+	// The one exception: the operator's release 6 hand path (novox/hq ADR 0539 clauses 2 and 10).
+	hand, _ := HandPathWrites(nil)
 	for _, p := range perms.Publish {
+		if slices.Contains(hand, p) {
+			continue
+		}
 		// A tool call — named by its caller or not (novox/hq issue 365) — or asking what answers (novox/hq
 		// ADR 0197), a question every service answers about itself, which claims nothing and controls nothing.
 		if !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") && !strings.HasPrefix(p, "$SRV.") {

internal/broker/streams.go

@@ -373,15 +373,15 @@ var ControllerStates = []string{"applied", "refused", "built-before",
 	// And a pull request's merge check, judged (novox/hq to-be 45 §9): what the forge's holder sets as
 	// the pull request's status, and anybody else may read.
 	"checked",
-	// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
-	"plan-moved",
-	// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
-	"sent",
+	// (A walk kept in a new state and a tier sent to a machine are gone with the plans, novox/hq ADR 0539.)
 	// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
 	// each registered module's declared config (design 57 F3).
 	"assignment-changed", "declared-config-changed", "predefined-changed",
 	// And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937).
-	"artifacts-gone"}
+	"artifacts-gone",
+	// And a bucket of the plans' state deleted at the controller's first start, and the declarations' stream once no node
+	// reads one (novox/hq ADR 0539 clauses 11 and 13).
+	"bucket-deleted"}
 
 // BusAdvisories are what the bus server says about the mesh's own account that the controller
 // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.bucket-deleted", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
@@ -39,7 +39,7 @@ accounts {
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
         publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_composed-one.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_composed-one.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_composed-one.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_composed-one", "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_composed-one.>", "$JS.FC.KV_mesh-delivery-assigner_wants.>", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>", "mesh.seat.node-backup.call.backed-up.one.node~one", "mesh.seat.node-backup.call.now.one.node~one", "mesh.seat.node-backup.tool.backed-up.one", "mesh.seat.node-backup.tool.now.one"] }
-        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.*.event.backed-up", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
+        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.restic.event.backed-up", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
       { user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {

internal/broker/users.go

@@ -230,7 +230,12 @@ func Users(r Records) ([]Principal, error) {
 		out = append(out, Principal{Kind: KindEnrolment, Node: node})
 	}
 	for _, person := range sortedNames(r.People) {
-		out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
+		p := Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]}
+		if administers(p.Invokes) {
+			// The operator's hand path writes each node's compositions (ADR 0539 clause 10).
+			p.FilesFor = sortedCopy(r.Nodes)
+		}
+		out = append(out, p)
 	}
 
 	// Refused here rather than discovered by the server. Two users with one name is a file the

internal/broker/writers.go

@@ -151,7 +151,7 @@ var WritersTable = []WriterRow{
 	{State: "the compositions and their digests", Writer: "controller (lease holder), compare-and-set",
 		KeptIn:   "key-value composed-<node> for each node, and " + delivery.ControllerComposedBucket,
 		Others:   "a node-engine reads its own node's; the assigner reads the digests",
-		Subjects: kvOf(delivery.ControllerComposedBucket), Writes: isController},
+		Subjects: kvOf(delivery.ControllerComposedBucket), Writes: isControllerOrTheHandPath},
 	{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
 		ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
 		"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
@@ -236,7 +236,7 @@ var WritersTable = []WriterRow{
 		KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
 			"node's latest declaration no longer names it",
 		Others:   "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
-		Subjects: []string{"$O.*.>"}, Writes: isController},
+		Subjects: []string{"$O.*.>"}, Writes: isControllerOrTheHandPath},
 	{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
 		KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
 		Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
@@ -299,3 +299,9 @@ func SubjectsOverlap(a, b string) bool {
 		}
 	}
 }
+
+// isControllerOrTheHandPath is the controller, or the operator's account (a person who may invoke every tool) writing
+// release 6's hand path: the one second writer ADR 0539 clauses 2 and 10 name, for a composition made by hand.
+func isControllerOrTheHandPath(p Principal, pattern string) bool {
+	return isController(p, pattern) || (p.Kind == KindPerson && administers(p.Invokes))
+}

internal/link/events.go

@@ -73,15 +73,7 @@ const (
 	// KeyRolledBack: a build failed its gate on its first machine and was put back there, or could not
 	// be (novox/hq ADR 0236, to-be 45 §8); or a witness on a machine put a core component back.
 	KeyRolledBack = "rolled-back"
-	// KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier,
-	// the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not
-	// wait for it.
-	KeySent = "sent"
-	// KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines —
-	// was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its
-	// steps from. Said after every save of the serving controller; a delivery's holder also asks for the
-	// walks it follows, so a save made elsewhere is found by comparison, never lost.
-	KeyPlanMoved = "plan-moved"
+	// (No plan is kept or sent any more, novox/hq ADR 0539: the events plan-moved and sent are gone with them.)
 	// KeyAssignmentChanged and KeyDeclaredConfigChanged: a key of the controller's assignments or declared-config
 	// mirror was written or removed (novox/hq ADR 0289, design 54 §3, design 57 F3): which key, how and at which
 	// revision of the bucket — never the value, which the bucket holds.