Return a grant not written and retry the take-in under a condition; drop the plans' events; grant bucket-deleted, restic's backup word and the operator's hand path (issue 1945; the review's D1, F1, H1 and its notes)

With no send left nothing else writes a missed grant, so the take-in of a done key returns it and tries again every
half minute under an urgent condition until it is written. plan-moved and sent go with the plans. The controller's
first-start deletions are said on a granted subject, a node hears only its backup holder, and the operator's account
writes the hand path, an object replaced included (ADR 0539 clauses 2, 7, 10 and 13).

cmd/mesh-controller/composed_keys.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -114,26 +116,64 @@ func inList(list []string, s string) bool {
 // doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
 var doneKeysMu sync.Map
 
-// takeDoneKey is a woken module's take-in on the live stores and buckets, said.
+// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
+// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
+// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
+// yet is no failure: its next write wakes this again.
 func takeDoneKey(ctx context.Context, open *stores, module string) {
 	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
 	mu.(*sync.Mutex).Lock()
 	defer mu.(*sync.Mutex).Unlock()
+	failed := false
+	for {
+		err := takeDoneKeyOnce(ctx, open, module)
+		o := doneKeyObservation(module)
+		if err == nil || errors.Is(err, errKeyWaits) {
+			if keeper := conditionsFrom; keeper != nil && failed {
+				_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
+			}
+			return
+		}
+		fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry)
+		if keeper := conditionsFrom; keeper != nil && !failed {
+			o.Summary += err.Error()
+			if _, oerr := keeper.Observe(ctx, o); oerr == nil {
+				failed = true
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(doneKeyRetry):
+		}
+	}
+}
+
+// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
+var doneKeyRetry = 30 * time.Second
+
+// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
+// they are, no want can name it.
+const kindNotComposed = "fingerprint-not-composed"
+
+func doneKeyObservation(module string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
+		Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
+		Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
+}
+
+// takeDoneKeyOnce is one take-in of a module's done key.
+func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
 	keys, closeKeys, err := openBuilderKeys()
 	if err != nil {
-		fmt.Printf("done keys: %s: the builder's keys cannot be read: %v\n", module, err)
-		return
+		return fmt.Errorf("the builder's keys cannot be read: %w", err)
 	}
 	defer closeKeys()
 	said, err := liveDoneKeySteps(open, keys).take(ctx, module)
-	switch {
-	case errors.Is(err, errKeyWaits):
-		// Said by nobody: the key's next write wakes this again.
-	case err != nil:
-		fmt.Printf("done keys: %s: %v\n", module, err)
-	case said != "":
+	if err == nil && said != "" {
 		fmt.Printf("done keys: %s\n", said)
 	}
+	return err
 }
 
 // liveDoneKeySteps are the steps on the controller's own stores and the bus.
@@ -197,8 +237,7 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
 			if err != nil {
 				return err
 			}
-			grantsIssued(ctx, inv, js, "done keys: ")
-			return nil
+			return grantsIssued(ctx, inv, js, "done keys: ")
 		},
 	}
 }