Give the controller seat the verb held and the event artifacts-gone, so the builder rebuilds a done key whose blob the artifact store no longer holds (issue 1937)

The store's nightly collector deletes every bare blob no manifest names, and a done key the controller never took in holds
nothing: node-tools, mesh-host and the forge bridge kept done keys naming blobs that answer 404. held lets the builder ask
before it reuses a key; artifacts-gone is said when a take-in finds a gone blob. delivery/held.go is the shared contract.

go/catalogue/seats.go

@@ -115,7 +115,10 @@ var defaultSeats = append([]Seat{
 			// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
 			"sent",
 			// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
-			"assignment-changed", "declared-config-changed", "predefined-changed"},
+			"assignment-changed", "declared-config-changed", "predefined-changed",
+			// A builder's key not taken in because the artifact store does not hold an artifact it names (novox/hq
+			// issue 1937): the builder builds it again.
+			"artifacts-gone"},
 		Serves: ControllerVerbs},
 	// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
 	// served by whichever module holds the seat with tools of these names.

go/catalogue/verbs.go

@@ -566,6 +566,15 @@ var ControllerVerbs = []Verb{
 			"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
 			"collected":  "\"true\": list the collected references too (they only grow; counted always)",
 		}, nil, "collected")},
+	// Whether the artifact store holds what a builder's done key names, before the builder reuses it (novox/hq issue
+	// 1937): the store's collector deletes a blob nothing holds, and a key the controller never took in holds nothing.
+	{Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " +
+		"held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key, and " +
+		"builds a key whose artifact is gone again. Reads only: nothing is held or written (novox/hq issue 1937).",
+		Input: listed(schema(map[string]string{
+			"references": "the artifacts' references, artifact-store://<module>/<artifact>/blobs/sha256:<hex>: a list, " +
+				"or one text separated by commas",
+		}, []string{"references"}), "references")},
 	{Name: "collect", Description: "The artifact store's sweep, now: what the mesh made and keeps for no reason " +
 		"is let go of. Without confirm a dry run — every kept archive asked about with HEADs, what would be let go " +
 		"listed, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible " +

go/delivery/held.go

@@ -0,0 +1,104 @@
+package delivery
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"time"
+)
+
+// Whether the artifact store still holds what a done key names (novox/hq issue 1937). A build module pushes a bundle or
+// an archive as a bare blob (ADR 0498 §1), and the store's nightly collector deletes every blob no manifest names; only
+// the controller's take-in and sweep hold one by its manifest. A done key the controller has not taken in can therefore
+// name a blob that is gone, and reusing it sends a definition every machine fails on. So the builder asks before it
+// reuses a done key, and the controller says when a take-in found a gone blob.
+const (
+	// HeldVerb is the controller seat's verb the builder asks, as a module's invokes names it: references in, each
+	// answered held or gone.
+	HeldVerb = "seat:mesh-controller.held"
+	// ArtifactsGoneEvent is the controller seat's event said when a take-in is refused because the store does not hold
+	// an artifact the key names: as a consumer names it, `mesh-controller.artifacts-gone`.
+	ArtifactsGoneEvent = "artifacts-gone"
+	// ArtifactsGoneSubject is where it is published (the seat's event subject).
+	ArtifactsGoneSubject = "mesh.seat.mesh-controller.event." + ArtifactsGoneEvent
+)
+
+// HeldAnswer is the answer of HeldVerb: the references the store holds, and those it answered 404 for. A reference the
+// store could not be asked about is in neither, and named in Unknown with why.
+type HeldAnswer struct {
+	Store   string            `json:"store,omitempty"`
+	Held    []string          `json:"held"`
+	Gone    []string          `json:"gone"`
+	Unknown map[string]string `json:"unknown,omitempty"`
+}
+
+// DecodeHeldAnswer reads HeldVerb's answer: the verb answers {output, ok, answer}, the answer being HeldAnswer. A
+// reference asked that the answer names in none of held, gone and unknown is an error: an answer that leaves one out
+// would let a key with a gone blob be reused.
+func DecodeHeldAnswer(raw []byte, asked []string) (HeldAnswer, error) {
+	var wrapped struct {
+		Output string          `json:"output"`
+		OK     bool            `json:"ok"`
+		Answer json.RawMessage `json:"answer"`
+	}
+	if err := json.Unmarshal(raw, &wrapped); err != nil {
+		return HeldAnswer{}, fmt.Errorf("the answer of %s is not {output, ok, answer}: %w", HeldVerb, err)
+	}
+	if !wrapped.OK || len(wrapped.Answer) == 0 {
+		return HeldAnswer{}, fmt.Errorf("%s did not answer: %s", HeldVerb, wrapped.Output)
+	}
+	var a HeldAnswer
+	if err := json.Unmarshal(wrapped.Answer, &a); err != nil {
+		return HeldAnswer{}, fmt.Errorf("the answer of %s is not {held, gone}: %w", HeldVerb, err)
+	}
+	named := map[string]bool{}
+	for _, r := range a.Held {
+		named[r] = true
+	}
+	for _, r := range a.Gone {
+		named[r] = true
+	}
+	for r := range a.Unknown {
+		named[r] = true
+	}
+	for _, r := range asked {
+		if !named[r] {
+			return HeldAnswer{}, fmt.Errorf("the answer of %s names nothing of %s", HeldVerb, r)
+		}
+	}
+	return a, nil
+}
+
+// ArtifactsGone is the body of ArtifactsGoneEvent: the module whose builder's key the controller would not take in,
+// the key and its fingerprint, and every reference of it the store does not hold.
+type ArtifactsGone struct {
+	Event       string    `json:"event"`
+	Module      string    `json:"module"`
+	Key         string    `json:"key"`
+	Fingerprint string    `json:"fingerprint"`
+	Gone        []string  `json:"gone"`
+	At          time.Time `json:"at"`
+}
+
+// EncodeArtifactsGone is the event's body, the references sorted.
+func EncodeArtifactsGone(e ArtifactsGone) ([]byte, error) {
+	e.Event = ArtifactsGoneEvent
+	e.Gone = append([]string(nil), e.Gone...)
+	sort.Strings(e.Gone)
+	if e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 {
+		return nil, fmt.Errorf("delivery: %s names a module, a fingerprint and at least one gone reference", ArtifactsGoneEvent)
+	}
+	return json.Marshal(e)
+}
+
+// DecodeArtifactsGone reads the event's body, refusing one that names no module, fingerprint or reference.
+func DecodeArtifactsGone(raw []byte) (ArtifactsGone, error) {
+	var e ArtifactsGone
+	if err := json.Unmarshal(raw, &e); err != nil {
+		return ArtifactsGone{}, fmt.Errorf("delivery: %s is not one: %w", ArtifactsGoneEvent, err)
+	}
+	if e.Event != ArtifactsGoneEvent || e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 {
+		return ArtifactsGone{}, fmt.Errorf("delivery: %s names no module, fingerprint or gone reference: %s", ArtifactsGoneEvent, raw)
+	}
+	return e, nil
+}

go/delivery/held_test.go

@@ -0,0 +1,84 @@
+package delivery
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+const (
+	refA = "artifact-store://node-tools/runtime/blobs/sha256:a46b5d51a30e8fe71107d75ceb46e3187022fe51f551d0498b8be817f22435cb"
+	refB = "artifact-store://mesh-host/host-arch/blobs/sha256:8a752f914f3523904a5442cae37ce3cb1ad297d6789b0f871585bce4d0c2ea00"
+)
+
+// 1937/AC1 (the contract): the controller's answer reads into held and gone, and an answer leaving a reference out is refused.
+func TestDecodeHeldAnswer(t *testing.T) {
+	raw := []byte(`{"output":"…","ok":true,"answer":{"store":"s:5100","held":["` + refA + `"],"gone":["` + refB + `"]}}`)
+	a, err := DecodeHeldAnswer(raw, []string{refA, refB})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(a.Held) != 1 || a.Held[0] != refA || len(a.Gone) != 1 || a.Gone[0] != refB {
+		t.Fatalf("read %+v", a)
+	}
+	if _, err := DecodeHeldAnswer(raw, []string{refA, refB, "artifact-store://x/y/blobs/sha256:00"}); err == nil ||
+		!strings.Contains(err.Error(), "names nothing of") {
+		t.Fatalf("an answer leaving a reference out was taken: %v", err)
+	}
+	if _, err := DecodeHeldAnswer([]byte(`{"output":"the store is down","ok":false}`), nil); err == nil ||
+		!strings.Contains(err.Error(), "the store is down") {
+		t.Fatalf("a failed verb was taken: %v", err)
+	}
+}
+
+// 1937/AC2 (the contract): the event's body names the module, the fingerprint and the gone references, sorted.
+func TestArtifactsGoneRoundTrip(t *testing.T) {
+	raw, err := EncodeArtifactsGone(ArtifactsGone{Module: "node-tools", Key: "node-tools.75", Fingerprint: "src2:75",
+		Gone: []string{refB, refA}, At: time.Unix(0, 0).UTC()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	e, err := DecodeArtifactsGone(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e.Event != ArtifactsGoneEvent || e.Module != "node-tools" || len(e.Gone) != 2 || e.Gone[0] != refB {
+		t.Fatalf("read %+v", e)
+	}
+	if _, err := EncodeArtifactsGone(ArtifactsGone{Module: "m", Fingerprint: "src2:1"}); err == nil {
+		t.Fatal("an event naming no gone reference was made")
+	}
+	if _, err := DecodeArtifactsGone([]byte(`{"event":"artifacts-gone","module":"m"}`)); err == nil {
+		t.Fatal("an event naming no fingerprint was read")
+	}
+}
+
+// The verb and the event are on the controller seat's contract under the names the builder uses.
+func TestHeldOnTheControllerSeat(t *testing.T) {
+	if HeldVerb != "seat:"+catalogue.ControllerSeatName+".held" {
+		t.Fatalf("HeldVerb %q", HeldVerb)
+	}
+	found := false
+	for _, v := range catalogue.ControllerVerbs {
+		if v.Name == "held" {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatal("the controller seat declares no verb held")
+	}
+	for _, s := range catalogue.DefaultSeats() {
+		if s.Name != catalogue.ControllerSeatName {
+			continue
+		}
+		for _, e := range s.Emits {
+			if e == ArtifactsGoneEvent {
+				return
+			}
+		}
+		t.Fatalf("the controller seat emits %v, not %s", s.Emits, ArtifactsGoneEvent)
+	}
+	t.Fatal("no controller seat")
+}