Return a grant not written and retry the take-in under a condition; drop the plans' events; grant bucket-deleted, restic's backup word and the operator's hand path (issue 1945; the review's D1, F1, H1 and its notes)

With no send left nothing else writes a missed grant, so the take-in of a done key returns it and tries again every
half minute under an urgent condition until it is written. plan-moved and sent go with the plans. The controller's
first-start deletions are said on a granted subject, a node hears only its backup holder, and the operator's account
writes the hand path, an object replaced included (ADR 0539 clauses 2, 7, 10 and 13).

cmd/mesh-controller/composed_keys.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -114,26 +116,64 @@ func inList(list []string, s string) bool {
 // doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
 var doneKeysMu sync.Map
 
-// takeDoneKey is a woken module's take-in on the live stores and buckets, said.
+// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
+// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
+// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
+// yet is no failure: its next write wakes this again.
 func takeDoneKey(ctx context.Context, open *stores, module string) {
 	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
 	mu.(*sync.Mutex).Lock()
 	defer mu.(*sync.Mutex).Unlock()
+	failed := false
+	for {
+		err := takeDoneKeyOnce(ctx, open, module)
+		o := doneKeyObservation(module)
+		if err == nil || errors.Is(err, errKeyWaits) {
+			if keeper := conditionsFrom; keeper != nil && failed {
+				_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
+			}
+			return
+		}
+		fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry)
+		if keeper := conditionsFrom; keeper != nil && !failed {
+			o.Summary += err.Error()
+			if _, oerr := keeper.Observe(ctx, o); oerr == nil {
+				failed = true
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(doneKeyRetry):
+		}
+	}
+}
+
+// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
+var doneKeyRetry = 30 * time.Second
+
+// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
+// they are, no want can name it.
+const kindNotComposed = "fingerprint-not-composed"
+
+func doneKeyObservation(module string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
+		Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
+		Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
+}
+
+// takeDoneKeyOnce is one take-in of a module's done key.
+func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
 	keys, closeKeys, err := openBuilderKeys()
 	if err != nil {
-		fmt.Printf("done keys: %s: the builder's keys cannot be read: %v\n", module, err)
-		return
+		return fmt.Errorf("the builder's keys cannot be read: %w", err)
 	}
 	defer closeKeys()
 	said, err := liveDoneKeySteps(open, keys).take(ctx, module)
-	switch {
-	case errors.Is(err, errKeyWaits):
-		// Said by nobody: the key's next write wakes this again.
-	case err != nil:
-		fmt.Printf("done keys: %s: %v\n", module, err)
-	case said != "":
+	if err == nil && said != "" {
 		fmt.Printf("done keys: %s\n", said)
 	}
+	return err
 }
 
 // liveDoneKeySteps are the steps on the controller's own stores and the bus.
@@ -197,8 +237,7 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
 			if err != nil {
 				return err
 			}
-			grantsIssued(ctx, inv, js, "done keys: ")
-			return nil
+			return grantsIssued(ctx, inv, js, "done keys: ")
 		},
 	}
 }

cmd/mesh-controller/composed_keys_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"encoding/json"
 	"strings"
 	"testing"
@@ -114,3 +115,15 @@ func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testin
 		}
 	}
 }
+
+// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left,
+// nothing else would write it, and no want could name the fingerprint.
+func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) {
+	fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
+	var acts []string
+	s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox")
+	s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") }
+	if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") {
+		t.Errorf("a grant not written: %v", err)
+	}
+}

cmd/mesh-controller/grants.go

@@ -126,28 +126,25 @@ func withVersion(g sdkdelivery.ControllerGrant) sdkdelivery.ControllerGrant {
 	return g
 }
 
-// grantsIssued writes the grants the bus's user list just sent to its machine issued (ADR 0528 clause 2). Said, never
-// fatal: a grant not written now is written at the next send of the list, and the assigner's wants wait for it.
-func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) {
+// grantsIssued writes the grants issued for every module whose credentials all exist at the fingerprint it is
+// registered at (ADR 0528 clause 2, ADR 0539 clause 7). Its error is the caller's: with no send left, nothing writes a
+// missed grant later but the caller's retry (the done key's take-in, composed_keys.go), and the assigner's wants wait.
+func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) error {
 	kv, err := js.KeyValue(ctx, broker.GrantsBucket)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: the bucket %s: %v\n", indent, broker.GrantsBucket, err)
-		return
+		return fmt.Errorf("the grants issued are not written: the bucket %s: %w", broker.GrantsBucket, err)
 	}
 	records, err := inv.BusRecords(ctx)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	principals, err := broker.Users(records)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	kept, err := inv.BusUsers(ctx)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	hashes := make(map[string]string, len(kept))
 	for name, u := range kept {
@@ -155,14 +152,14 @@ func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.Je
 	}
 	current, err := currentFingerprints(ctx, inv)
 	if err != nil {
-		fmt.Printf("%sthe grants issued are not written: %v\n", indent, err)
-		return
+		return fmt.Errorf("the grants issued are not written: %w", err)
 	}
 	wrote, err := keepGrants(ctx, kv, grantsFrom(principals, hashes, current))
 	if wrote > 0 {
 		fmt.Printf("%s%d grant(s) issued written for the assigner\n", indent, wrote)
 	}
 	if err != nil {
-		fmt.Printf("%ssome grants issued are not written: %v\n", indent, err)
+		return fmt.Errorf("some grants issued are not written: %w", err)
 	}
+	return nil
 }