From 34f210e5eba1d16abe06b526de1026c282692b83 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 04:07:22 +0200 Subject: [PATCH] Give the controller seat the verb held and the event artifacts-gone, so the builder rebuilds a done key whose blob the artifact store no longer holds (issue 1937) The store's nightly collector deletes every bare blob no manifest names, and a done key the controller never took in holds nothing: node-tools, mesh-host and the forge bridge kept done keys naming blobs that answer 404. held lets the builder ask before it reuses a key; artifacts-gone is said when a take-in finds a gone blob. delivery/held.go is the shared contract. --- go/catalogue/seats.go | 5 +- go/catalogue/verbs.go | 9 ++++ go/delivery/held.go | 104 +++++++++++++++++++++++++++++++++++++++ go/delivery/held_test.go | 84 +++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 go/delivery/held.go create mode 100644 go/delivery/held_test.go diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 05d4961..a22827c 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -115,7 +115,10 @@ var defaultSeats = append([]Seat{ // A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). "sent", // A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3). - "assignment-changed", "declared-config-changed", "predefined-changed"}, + "assignment-changed", "declared-config-changed", "predefined-changed", + // A builder's key not taken in because the artifact store does not hold an artifact it names (novox/hq + // issue 1937): the builder builds it again. + "artifacts-gone"}, Serves: ControllerVerbs}, // The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable, // served by whichever module holds the seat with tools of these names. diff --git a/go/catalogue/verbs.go b/go/catalogue/verbs.go index bdf299b..aad0f6d 100644 --- a/go/catalogue/verbs.go +++ b/go/catalogue/verbs.go @@ -566,6 +566,15 @@ var ControllerVerbs = []Verb{ "repository": "only this repository's references, as / (the counts stay whole)", "collected": "\"true\": list the collected references too (they only grow; counted always)", }, nil, "collected")}, + // Whether the artifact store holds what a builder's done key names, before the builder reuses it (novox/hq issue + // 1937): the store's collector deletes a blob nothing holds, and a key the controller never took in holds nothing. + {Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " + + "held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key, and " + + "builds a key whose artifact is gone again. Reads only: nothing is held or written (novox/hq issue 1937).", + Input: listed(schema(map[string]string{ + "references": "the artifacts' references, artifact-store:////blobs/sha256:: a list, " + + "or one text separated by commas", + }, []string{"references"}), "references")}, {Name: "collect", Description: "The artifact store's sweep, now: what the mesh made and keeps for no reason " + "is let go of. Without confirm a dry run — every kept archive asked about with HEADs, what would be let go " + "listed, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible " + diff --git a/go/delivery/held.go b/go/delivery/held.go new file mode 100644 index 0000000..77e243a --- /dev/null +++ b/go/delivery/held.go @@ -0,0 +1,104 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "sort" + "time" +) + +// Whether the artifact store still holds what a done key names (novox/hq issue 1937). A build module pushes a bundle or +// an archive as a bare blob (ADR 0498 §1), and the store's nightly collector deletes every blob no manifest names; only +// the controller's take-in and sweep hold one by its manifest. A done key the controller has not taken in can therefore +// name a blob that is gone, and reusing it sends a definition every machine fails on. So the builder asks before it +// reuses a done key, and the controller says when a take-in found a gone blob. +const ( + // HeldVerb is the controller seat's verb the builder asks, as a module's invokes names it: references in, each + // answered held or gone. + HeldVerb = "seat:mesh-controller.held" + // ArtifactsGoneEvent is the controller seat's event said when a take-in is refused because the store does not hold + // an artifact the key names: as a consumer names it, `mesh-controller.artifacts-gone`. + ArtifactsGoneEvent = "artifacts-gone" + // ArtifactsGoneSubject is where it is published (the seat's event subject). + ArtifactsGoneSubject = "mesh.seat.mesh-controller.event." + ArtifactsGoneEvent +) + +// HeldAnswer is the answer of HeldVerb: the references the store holds, and those it answered 404 for. A reference the +// store could not be asked about is in neither, and named in Unknown with why. +type HeldAnswer struct { + Store string `json:"store,omitempty"` + Held []string `json:"held"` + Gone []string `json:"gone"` + Unknown map[string]string `json:"unknown,omitempty"` +} + +// DecodeHeldAnswer reads HeldVerb's answer: the verb answers {output, ok, answer}, the answer being HeldAnswer. A +// reference asked that the answer names in none of held, gone and unknown is an error: an answer that leaves one out +// would let a key with a gone blob be reused. +func DecodeHeldAnswer(raw []byte, asked []string) (HeldAnswer, error) { + var wrapped struct { + Output string `json:"output"` + OK bool `json:"ok"` + Answer json.RawMessage `json:"answer"` + } + if err := json.Unmarshal(raw, &wrapped); err != nil { + return HeldAnswer{}, fmt.Errorf("the answer of %s is not {output, ok, answer}: %w", HeldVerb, err) + } + if !wrapped.OK || len(wrapped.Answer) == 0 { + return HeldAnswer{}, fmt.Errorf("%s did not answer: %s", HeldVerb, wrapped.Output) + } + var a HeldAnswer + if err := json.Unmarshal(wrapped.Answer, &a); err != nil { + return HeldAnswer{}, fmt.Errorf("the answer of %s is not {held, gone}: %w", HeldVerb, err) + } + named := map[string]bool{} + for _, r := range a.Held { + named[r] = true + } + for _, r := range a.Gone { + named[r] = true + } + for r := range a.Unknown { + named[r] = true + } + for _, r := range asked { + if !named[r] { + return HeldAnswer{}, fmt.Errorf("the answer of %s names nothing of %s", HeldVerb, r) + } + } + return a, nil +} + +// ArtifactsGone is the body of ArtifactsGoneEvent: the module whose builder's key the controller would not take in, +// the key and its fingerprint, and every reference of it the store does not hold. +type ArtifactsGone struct { + Event string `json:"event"` + Module string `json:"module"` + Key string `json:"key"` + Fingerprint string `json:"fingerprint"` + Gone []string `json:"gone"` + At time.Time `json:"at"` +} + +// EncodeArtifactsGone is the event's body, the references sorted. +func EncodeArtifactsGone(e ArtifactsGone) ([]byte, error) { + e.Event = ArtifactsGoneEvent + e.Gone = append([]string(nil), e.Gone...) + sort.Strings(e.Gone) + if e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 { + return nil, fmt.Errorf("delivery: %s names a module, a fingerprint and at least one gone reference", ArtifactsGoneEvent) + } + return json.Marshal(e) +} + +// DecodeArtifactsGone reads the event's body, refusing one that names no module, fingerprint or reference. +func DecodeArtifactsGone(raw []byte) (ArtifactsGone, error) { + var e ArtifactsGone + if err := json.Unmarshal(raw, &e); err != nil { + return ArtifactsGone{}, fmt.Errorf("delivery: %s is not one: %w", ArtifactsGoneEvent, err) + } + if e.Event != ArtifactsGoneEvent || e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 { + return ArtifactsGone{}, fmt.Errorf("delivery: %s names no module, fingerprint or gone reference: %s", ArtifactsGoneEvent, raw) + } + return e, nil +} diff --git a/go/delivery/held_test.go b/go/delivery/held_test.go new file mode 100644 index 0000000..f5b71c4 --- /dev/null +++ b/go/delivery/held_test.go @@ -0,0 +1,84 @@ +package delivery + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +const ( + refA = "artifact-store://node-tools/runtime/blobs/sha256:a46b5d51a30e8fe71107d75ceb46e3187022fe51f551d0498b8be817f22435cb" + refB = "artifact-store://mesh-host/host-arch/blobs/sha256:8a752f914f3523904a5442cae37ce3cb1ad297d6789b0f871585bce4d0c2ea00" +) + +// 1937/AC1 (the contract): the controller's answer reads into held and gone, and an answer leaving a reference out is refused. +func TestDecodeHeldAnswer(t *testing.T) { + raw := []byte(`{"output":"…","ok":true,"answer":{"store":"s:5100","held":["` + refA + `"],"gone":["` + refB + `"]}}`) + a, err := DecodeHeldAnswer(raw, []string{refA, refB}) + if err != nil { + t.Fatal(err) + } + if len(a.Held) != 1 || a.Held[0] != refA || len(a.Gone) != 1 || a.Gone[0] != refB { + t.Fatalf("read %+v", a) + } + if _, err := DecodeHeldAnswer(raw, []string{refA, refB, "artifact-store://x/y/blobs/sha256:00"}); err == nil || + !strings.Contains(err.Error(), "names nothing of") { + t.Fatalf("an answer leaving a reference out was taken: %v", err) + } + if _, err := DecodeHeldAnswer([]byte(`{"output":"the store is down","ok":false}`), nil); err == nil || + !strings.Contains(err.Error(), "the store is down") { + t.Fatalf("a failed verb was taken: %v", err) + } +} + +// 1937/AC2 (the contract): the event's body names the module, the fingerprint and the gone references, sorted. +func TestArtifactsGoneRoundTrip(t *testing.T) { + raw, err := EncodeArtifactsGone(ArtifactsGone{Module: "node-tools", Key: "node-tools.75", Fingerprint: "src2:75", + Gone: []string{refB, refA}, At: time.Unix(0, 0).UTC()}) + if err != nil { + t.Fatal(err) + } + e, err := DecodeArtifactsGone(raw) + if err != nil { + t.Fatal(err) + } + if e.Event != ArtifactsGoneEvent || e.Module != "node-tools" || len(e.Gone) != 2 || e.Gone[0] != refB { + t.Fatalf("read %+v", e) + } + if _, err := EncodeArtifactsGone(ArtifactsGone{Module: "m", Fingerprint: "src2:1"}); err == nil { + t.Fatal("an event naming no gone reference was made") + } + if _, err := DecodeArtifactsGone([]byte(`{"event":"artifacts-gone","module":"m"}`)); err == nil { + t.Fatal("an event naming no fingerprint was read") + } +} + +// The verb and the event are on the controller seat's contract under the names the builder uses. +func TestHeldOnTheControllerSeat(t *testing.T) { + if HeldVerb != "seat:"+catalogue.ControllerSeatName+".held" { + t.Fatalf("HeldVerb %q", HeldVerb) + } + found := false + for _, v := range catalogue.ControllerVerbs { + if v.Name == "held" { + found = true + } + } + if !found { + t.Fatal("the controller seat declares no verb held") + } + for _, s := range catalogue.DefaultSeats() { + if s.Name != catalogue.ControllerSeatName { + continue + } + for _, e := range s.Emits { + if e == ArtifactsGoneEvent { + return + } + } + t.Fatalf("the controller seat emits %v, not %s", s.Emits, ArtifactsGoneEvent) + } + t.Fatal("no controller seat") +}