diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index 166ab10c..dd67b04f 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -266,6 +266,12 @@ func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool { ref, err := v.Mirror(holder.ImageSeat, stdio.CallerOf(args), r) return map[string]any{"reference": ref}, err }}, + stdio.Tool{Name: "registry_health", Description: "distribution's health check, run by the runtime each minute: versions of each seat asked through the store's read side (the program's port 5000); fails naming the program when it does not answer.", + Input: obj(map[string]any{}), + Run: func(map[string]any) (any, error) { + front, _ := url.Parse("http://127.0.0.1:5000") + return holder.Health(&holder.OCI{Base: front}, finding) + }}, stdio.Tool{Name: "registry_incoming", Description: "Open transfers of incoming and its fill; with drop, drops one by hand (a hand act, why required).", Input: obj(map[string]any{"drop": s("a transfer id to drop"), "why": s("why it is dropped")}), Run: func(args map[string]any) (any, error) { diff --git a/modules/distribution/internal/holder/health.go b/modules/distribution/internal/holder/health.go new file mode 100644 index 00000000..1473e563 --- /dev/null +++ b/modules/distribution/internal/holder/health.go @@ -0,0 +1,34 @@ +package holder + +import "fmt" + +// FindingReadSide is the finding of a store whose read side does not answer through the program. +const FindingReadSide = "read-side-down" + +// ProbeName is the name the health check lists on each seat: any name answers, an empty list included. +const ProbeName = Self + "/program" + +// Health is distribution's health check (design 51 *While the program is down*; ADR 0240), run by the runtime each +// minute outside the program's process: it asks `versions` of each seat it holds through the store's read side, the +// program's port (front), and answers what failed, naming the program. The finding is written while it fails and +// cleared once it answers. +func Health(front Catalog, finding func(kind, subject, words string, raised bool)) (map[string]any, error) { + answered := map[string]int{} + for _, seat := range Seats { + got, err := front.Versions(ProbeName) + if err != nil { + words := fmt.Sprintf("the store's read side does not answer %s's versions through distribution's program (port 5000): %v; is the program distribution running?", seat, err) + if finding != nil { + finding(FindingReadSide, seat, words, true) + } + return nil, fmt.Errorf("%s", words) + } + answered[seat] = len(got) + } + if finding != nil { + for _, seat := range Seats { + finding(FindingReadSide, seat, "", false) + } + } + return map[string]any{"healthy": true, "versions": answered}, nil +} diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index a12da1e6..e910e671 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -477,6 +477,18 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { t.Error("the finding stays after the window closed") } }) + t.Run("the health check: versions of each seat through the program's port; down names the program, then clears", func(t *testing.T) { + front := &downCatalog{fakeCatalog: fakeCatalog{tags: map[string]string{}}, down: true} + found := map[string]bool{} + f := func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised } + if _, err := Health(front, f); err == nil || !strings.Contains(err.Error(), "program distribution") || !found[FindingReadSide+" "+ImageSeat] { + t.Errorf("program stopped: %v, findings %v", err, found) + } + front.down = false + if _, err := Health(front, f); err != nil || found[FindingReadSide+" "+ImageSeat] { + t.Errorf("program back: %v, findings %v", err, found) + } + }) t.Run("the manifest", func(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -489,6 +501,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { Serves []string `json:"serves"` } `json:"claims"` Resources []map[string]any `json:"resources"` + Consumes []string `json:"consumes"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -507,6 +520,9 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { if _, ok := claims[OldSeat]; !ok { t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open") } + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) { + t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes) + } // E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts. store, program := -1, -1 for i, r := range m.Resources { @@ -518,6 +534,9 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { } case "program": program = i + if h, _ := r["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "registry_health" { + t.Errorf("the program's health = %v, want the tool registry_health, run outside its process", r["health"]) + } } } if store < 0 || program < 0 || store > program { diff --git a/modules/distribution/internal/holder/mirror_fakes_test.go b/modules/distribution/internal/holder/mirror_fakes_test.go index c33b649d..33af195c 100644 --- a/modules/distribution/internal/holder/mirror_fakes_test.go +++ b/modules/distribution/internal/holder/mirror_fakes_test.go @@ -1,6 +1,7 @@ package holder import ( + "errors" "io" "net/http" "strings" @@ -106,3 +107,16 @@ type memWindows struct { func (m *memWindows) Window(n string) (Window, error) { return m.w[n], nil } func (m *memWindows) PutWindow(w Window) error { m.w[w.Name] = w; return nil } func (m *memWindows) Publishing() (Publishing, error) { return m.pub, nil } + +// downCatalog is a store whose read side does not answer, while down is set. +type downCatalog struct { + fakeCatalog + down bool +} + +func (d *downCatalog) Versions(name string) ([]string, error) { + if d.down { + return nil, errors.New("connection refused") + } + return d.fakeCatalog.Versions(name) +} diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 537ba8c6..29e3038c 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -44,6 +44,7 @@ ], "tools": [ "registry_direct_writes", + "registry_health", "registry_holds", "registry_incoming", "registry_old_name" @@ -198,8 +199,12 @@ }, "user": "distribution", "health": { - "kind": "http", - "url": "http://127.0.0.1:5000/v2/" + "kind": "tool", + "tool": "registry_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "90s" } }, {