Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).

modules/mesh-watcher/cmd/mesh-watcher/main.go

@@ -15,6 +15,8 @@ import (
 	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
@@ -98,8 +100,10 @@ func run(w *Watcher, l *listening, node string) {
 			time.Sleep(10 * time.Minute)
 		}
 	}()
+	follow := newTakeover()
+	go followTakeover(follow)
 	handle := func(e stdio.Envelope) error {
-		if !isHeartbeat(e.Key) {
+		if !follow.heard(e.Key) {
 			return nil
 		}
 		b, err := DecodeBeat(e.Body)
@@ -113,6 +117,9 @@ func run(w *Watcher, l *listening, node string) {
 	time.Sleep(500 * time.Millisecond)
 	for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
 		err := stdio.Subscribe(ControllerSeat+".*", handle)
+		if err == nil {
+			err = stdio.Subscribe(health.SelfCheckModule+".*", handle)
+		}
 		if err == nil {
 			l.set("listening")
 			logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node)
@@ -144,60 +151,46 @@ func roundTrip(node string) error {
 	}
 }
 
-// controlNodes reads which machines hold the controller and the bus.
+// controlNodes reads which machines hold the controller and the bus, from the placement seat's state of who holds
+// which seat (novox/hq ADR 0571 clause 4).
 func controlNodes() ([]string, error) {
-	raw, err := stdio.Ask("seat:mesh-controller.seats", map[string]any{})
+	st := stdio.State(delivery.SeatHoldersRead)
+	names, err := st.Keys()
 	if err != nil {
 		return nil, err
 	}
-	return holdersOf(raw, "mesh-controller", "mesh-broker")
+	keys := map[string][]byte{}
+	for _, k := range names {
+		e, err := st.Get(k)
+		if err != nil {
+			return nil, err
+		}
+		if e != nil {
+			keys[k] = e.Value
+		}
+	}
+	return holderNodes(keys, "mesh-controller", "mesh-broker")
 }
 
-// holdersOf reads the seats verb's answer, bare or inside a tool reply, for the named seats' machines.
-func holdersOf(raw json.RawMessage, seats ...string) ([]string, error) {
-	var answer struct {
-		Seats []struct {
-			Seat    string `json:"seat"`
-			Holders []struct {
-				Node string `json:"node"`
-			} `json:"holders"`
-		} `json:"seats"`
-		Output  string `json:"output"`
-		Content []struct {
-			Text string `json:"text"`
-		} `json:"content"`
-	}
-	if err := json.Unmarshal(raw, &answer); err != nil {
-		var s string
-		if json.Unmarshal(raw, &s) == nil {
-			return holdersOf(json.RawMessage(s), seats...)
-		}
-		return nil, fmt.Errorf("the seats answer is not JSON")
-	}
-	if len(answer.Seats) == 0 {
-		switch {
-		case answer.Output != "":
-			return holdersOf(json.RawMessage(answer.Output), seats...)
-		case len(answer.Content) > 0:
-			return holdersOf(json.RawMessage(answer.Content[0].Text), seats...)
-		}
-		return nil, fmt.Errorf("the seats answer lists no seat")
-	}
+// holderNodes is the machines holding any of the seats, in the order the seats are named, each once.
+func holderNodes(keys map[string][]byte, seats ...string) ([]string, error) {
 	var out []string
 	seen := map[string]bool{}
-	for _, s := range answer.Seats {
-		for _, want := range seats {
-			if s.Seat != want {
-				continue
-			}
-			for _, h := range s.Holders {
-				if !seen[h.Node] {
-					seen[h.Node] = true
-					out = append(out, h.Node)
-				}
+	for _, seat := range seats {
+		holders, err := delivery.HoldersOf(keys, seat, "")
+		if err != nil {
+			return nil, err
+		}
+		for _, h := range holders {
+			if !seen[h.Node] {
+				seen[h.Node] = true
+				out = append(out, h.Node)
 			}
 		}
 	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("the placement seat's state names no holder of %s", strings.Join(seats, " or "))
+	}
 	return out, nil
 }
 

modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go

@@ -34,10 +34,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
 	if err := json.Unmarshal(raw, &m); err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent}) {
+	if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent, "mesh-self-check.checked"}) {
 		t.Fatalf("consumes %v", m.Consumes)
 	}
-	if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping", "seat:mesh-controller.seats"}) {
+	if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping"}) {
 		t.Fatalf("invokes %v", m.Invokes)
 	}
 	if len(m.Claims) != 0 {

modules/mesh-watcher/cmd/mesh-watcher/takeover.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"sync"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// takeover is which self-check run the watcher hears: the controller's `doctor-heartbeat` until the keeper's
+// take-over key names the keeper, then mesh-self-check's `checked`, whose body is the same run (novox/hq ADR 0564
+// clause 9, issue 1952 W1). An absent key follows the controller.
+type takeover struct {
+	mu sync.Mutex
+	t  health.Takeover
+}
+
+func newTakeover() *takeover {
+	return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}}
+}
+
+// heard says whether key is the followed self-check event.
+func (f *takeover) heard(key string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	seat, event := f.t.SelfCheckEvent()
+	return key == seat+"."+event
+}
+
+func (f *takeover) set(value []byte, found bool) error {
+	t, err := health.DecodeTakeover(value, found)
+	if err != nil {
+		return err
+	}
+	f.mu.Lock()
+	f.t = t
+	f.mu.Unlock()
+	return nil
+}
+
+// followTakeover watches the key for the life of the process; one it cannot read leaves the event it heard.
+func followTakeover(f *takeover) {
+	for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
+		err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error {
+			if err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0); err != nil {
+				logf("[mesh-watcher] the take-over key is unreadable (%v); no change in what it hears", err)
+			} else {
+				logf("[mesh-watcher] the take-over key now follows %s", f.t.Follow)
+			}
+			return nil
+		})
+		if err == nil {
+			return
+		}
+		logf("[mesh-watcher] not watching the take-over key yet (%v)", err)
+		time.Sleep(wait)
+	}
+}

modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"errors"
 	"strings"
 	"testing"
@@ -214,16 +213,18 @@ func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) {
 	}
 }
 
-func TestTheSeatsAnswerIsReadInEitherShape(t *testing.T) {
-	bare := `{"seats":[{"seat":"mesh-controller","holders":[{"node":"n1","module":"mesh-controller"}]},
-		{"seat":"mesh-broker","holders":[{"node":"n1"},{"node":"n2"}]},{"seat":"git","holders":[{"node":"n3"}]}]}`
-	wrapped, _ := json.Marshal(map[string]any{"content": []map[string]string{{"type": "text", "text": bare}}})
-	output, _ := json.Marshal(map[string]any{"ok": true, "output": bare})
-	for _, raw := range []string{bare, string(wrapped), string(output)} {
-		got, err := holdersOf(json.RawMessage(raw), "mesh-controller", "mesh-broker")
-		if err != nil || strings.Join(got, ",") != "n1,n2" {
-			t.Errorf("%s: %v %v", raw, got, err)
-		}
+func TestTheHoldersAreReadFromThePlacementSeatsState(t *testing.T) {
+	keys := map[string][]byte{
+		"mesh-controller": []byte(`{"v":1,"seat":"mesh-controller","scope":"mesh","holders":[{"module":"mesh-controller","node":"n1"}]}`),
+		"mesh-broker":     []byte(`{"v":1,"seat":"mesh-broker","scope":"mesh","holders":[{"module":"nats","node":"n1"},{"module":"nats","node":"n2"}]}`),
+		"git":             []byte(`{"v":1,"seat":"git","scope":"mesh","holders":[{"module":"gitea","node":"n3"}]}`),
+	}
+	got, err := holderNodes(keys, "mesh-controller", "mesh-broker")
+	if err != nil || strings.Join(got, ",") != "n1,n2" {
+		t.Errorf("%v %v", got, err)
+	}
+	if _, err := holderNodes(map[string][]byte{}, "mesh-controller", "mesh-broker"); err == nil {
+		t.Error("no holder named, and no error")
 	}
 }