CAT-D6: distribution's health is a tool the runtime runs outside the program, asking versions through the store's read side

An http probe of /v2/ said only that a port answered; design 51 asks the health check to call versions per seat and
probe the read side each minute, naming the program when it is down, from outside its process. registry_health asks
versions of each seat through the program's port, writes the read-side finding while it fails and clears it after.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig

modules/distribution/cmd/distribution-tools/main.go

@@ -266,6 +266,12 @@ func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool {
 				ref, err := v.Mirror(holder.ImageSeat, stdio.CallerOf(args), r)
 				return map[string]any{"reference": ref}, err
 			}},
+		stdio.Tool{Name: "registry_health", Description: "distribution's health check, run by the runtime each minute: versions of each seat asked through the store's read side (the program's port 5000); fails naming the program when it does not answer.",
+			Input: obj(map[string]any{}),
+			Run: func(map[string]any) (any, error) {
+				front, _ := url.Parse("http://127.0.0.1:5000")
+				return holder.Health(&holder.OCI{Base: front}, finding)
+			}},
 		stdio.Tool{Name: "registry_incoming", Description: "Open transfers of incoming and its fill; with drop, drops one by hand (a hand act, why required).",
 			Input: obj(map[string]any{"drop": s("a transfer id to drop"), "why": s("why it is dropped")}),
 			Run: func(args map[string]any) (any, error) {

modules/distribution/internal/holder/health.go

@@ -0,0 +1,34 @@
+package holder
+
+import "fmt"
+
+// FindingReadSide is the finding of a store whose read side does not answer through the program.
+const FindingReadSide = "read-side-down"
+
+// ProbeName is the name the health check lists on each seat: any name answers, an empty list included.
+const ProbeName = Self + "/program"
+
+// Health is distribution's health check (design 51 *While the program is down*; ADR 0240), run by the runtime each
+// minute outside the program's process: it asks `versions` of each seat it holds through the store's read side, the
+// program's port (front), and answers what failed, naming the program. The finding is written while it fails and
+// cleared once it answers.
+func Health(front Catalog, finding func(kind, subject, words string, raised bool)) (map[string]any, error) {
+	answered := map[string]int{}
+	for _, seat := range Seats {
+		got, err := front.Versions(ProbeName)
+		if err != nil {
+			words := fmt.Sprintf("the store's read side does not answer %s's versions through distribution's program (port 5000): %v; is the program distribution running?", seat, err)
+			if finding != nil {
+				finding(FindingReadSide, seat, words, true)
+			}
+			return nil, fmt.Errorf("%s", words)
+		}
+		answered[seat] = len(got)
+	}
+	if finding != nil {
+		for _, seat := range Seats {
+			finding(FindingReadSide, seat, "", false)
+		}
+	}
+	return map[string]any{"healthy": true, "versions": answered}, nil
+}

modules/distribution/internal/holder/holder_test.go

@@ -477,6 +477,18 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
 			t.Error("the finding stays after the window closed")
 		}
 	})
+	t.Run("the health check: versions of each seat through the program's port; down names the program, then clears", func(t *testing.T) {
+		front := &downCatalog{fakeCatalog: fakeCatalog{tags: map[string]string{}}, down: true}
+		found := map[string]bool{}
+		f := func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }
+		if _, err := Health(front, f); err == nil || !strings.Contains(err.Error(), "program distribution") || !found[FindingReadSide+" "+ImageSeat] {
+			t.Errorf("program stopped: %v, findings %v", err, found)
+		}
+		front.down = false
+		if _, err := Health(front, f); err != nil || found[FindingReadSide+" "+ImageSeat] {
+			t.Errorf("program back: %v, findings %v", err, found)
+		}
+	})
 	t.Run("the manifest", func(t *testing.T) {
 		raw, err := os.ReadFile("../../module.json")
 		if err != nil {
@@ -489,6 +501,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
 				Serves []string `json:"serves"`
 			} `json:"claims"`
 			Resources []map[string]any `json:"resources"`
+			Consumes  []string         `json:"consumes"`
 		}
 		if err := json.Unmarshal(raw, &m); err != nil {
 			t.Fatal(err)
@@ -507,6 +520,9 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
 		if _, ok := claims[OldSeat]; !ok {
 			t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open")
 		}
+		if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) {
+			t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes)
+		}
 		// E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts.
 		store, program := -1, -1
 		for i, r := range m.Resources {
@@ -518,6 +534,9 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
 				}
 			case "program":
 				program = i
+				if h, _ := r["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "registry_health" {
+					t.Errorf("the program's health = %v, want the tool registry_health, run outside its process", r["health"])
+				}
 			}
 		}
 		if store < 0 || program < 0 || store > program {

modules/distribution/internal/holder/mirror_fakes_test.go

@@ -1,6 +1,7 @@
 package holder
 
 import (
+	"errors"
 	"io"
 	"net/http"
 	"strings"
@@ -106,3 +107,16 @@ type memWindows struct {
 func (m *memWindows) Window(n string) (Window, error) { return m.w[n], nil }
 func (m *memWindows) PutWindow(w Window) error        { m.w[w.Name] = w; return nil }
 func (m *memWindows) Publishing() (Publishing, error) { return m.pub, nil }
+
+// downCatalog is a store whose read side does not answer, while down is set.
+type downCatalog struct {
+	fakeCatalog
+	down bool
+}
+
+func (d *downCatalog) Versions(name string) ([]string, error) {
+	if d.down {
+		return nil, errors.New("connection refused")
+	}
+	return d.fakeCatalog.Versions(name)
+}

modules/distribution/module.json

@@ -44,6 +44,7 @@
   ],
   "tools": [
     "registry_direct_writes",
+    "registry_health",
     "registry_holds",
     "registry_incoming",
     "registry_old_name"
@@ -198,8 +199,12 @@
       },
       "user": "distribution",
       "health": {
-        "kind": "http",
-        "url": "http://127.0.0.1:5000/v2/"
+        "kind": "tool",
+        "tool": "registry_health",
+        "interval": "60s",
+        "timeout": "10s",
+        "looks": 2,
+        "grace": "90s"
       }
     },
     {