Give a channel's verified sender back when the root-free judgement passes again, and say when it is dropped (issue 930)

A push composed during a failed judgement issued the router a membership without the verified sender and nothing composed it again, so every approval was refused. The self-check now issues it again within one run; a composition says which kind lost it and why.
2026-10-16 17:42:23 +02:00
parent e231185572
commit b2e2bf0638
4 changed files with 330 additions and 13 deletions
@@ -0,0 +1,210 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A channel's verified sender is composed into a membership only while the machines are judged root-free at
+// that moment (novox/hq ADR 0259 §8), and the judgement fails closed when a read fails (novox/hq issue 930):
+// a push composed during a failed read issued the router a membership without it, and nothing composed it
+// again, so every approval was refused until somebody pushed. Two things follow.
+//
+// What a composition drops it says (droppedVerifiedSenders), with the reason the judgement gave.
+//
+// And the self-check issues it again (reissueVerifiedSenders): each time it judges the machines, a membership
+// that lacks a verified sender the composition would now give it is issued again with only its kinds changed.
+// **Bound: one self-check, doctorEvery (five minutes), after the judgement passes.** It only ever adds: a
+// machine that stops passing keeps what it was issued, as at a push, because the router asks the root-free
+// verb again before it honours an approval, and that is the check that holds.
+
+// droppedVerifiedSenders says, one line each, every kind that declares `verified-sender` and was composed
+// without it, naming the kind, its machine and why. whys is each machine's verdict reason (rootVerdict.Why).
+func droppedVerifiedSenders(r broker.Records, where broker.Placements, whys map[string]string) []string {
+	var routerNodes []string
+	for node, declared := range r.Assigned {
+		for _, d := range declared {
+			for _, s := range d.Holds {
+				if s.Name == routerSeat && !slices.Contains(routerNodes, node) {
+					routerNodes = append(routerNodes, node)
+				}
+			}
+		}
+	}
+	sort.Strings(routerNodes)
+	var out []string
+	for _, k := range where.Kinds {
+		declared := false
+		for _, d := range r.Assigned[k.Node] {
+			for _, s := range d.Holds {
+				if d.Module == k.Module && s.Name == k.Seat && s.Kind == k.Kind && slices.Contains(s.Capabilities, "verified-sender") {
+					declared = true
+				}
+			}
+		}
+		if !declared || slices.Contains(k.Capabilities, "verified-sender") {
+			continue
+		}
+		var why string
+		switch {
+		case !r.RootFree[k.Node]:
+			why = k.Node + " is not judged root-free: " + whys[k.Node]
+		case len(routerNodes) != 1:
+			why = fmt.Sprintf("the router is placed on %d machines, not one", len(routerNodes))
+		case !r.RootFree[routerNodes[0]]:
+			why = "the router's machine " + routerNodes[0] + " is not judged root-free: " + whys[routerNodes[0]]
+		default:
+			why = k.Module + " does not run as an account of its own"
+		}
+		out = append(out, fmt.Sprintf("  %s kind %s of %s on %s is composed without its verified sender: %s",
+			k.Seat, k.Kind, k.Module, k.Node, why))
+	}
+	return out
+}
+
+// withVerifiedSenders is a membership as issued with its seat traffic's kinds replaced by the composed ones,
+// and whether that changed it: only when a composed kind carries a verified sender the issued one lacks. Pure.
+func withVerifiedSenders(issued []byte, composed *broker.SeatTraffic) ([]byte, bool, error) {
+	if composed == nil {
+		return issued, false, nil
+	}
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal(issued, &fields); err != nil {
+		return nil, false, err
+	}
+	var traffic map[string]json.RawMessage
+	var had []broker.KindHeld
+	if raw, ok := fields["seat-traffic"]; ok {
+		if err := json.Unmarshal(raw, &traffic); err != nil {
+			return nil, false, err
+		}
+		if raw, ok := traffic["kinds"]; ok {
+			if err := json.Unmarshal(raw, &had); err != nil {
+				return nil, false, err
+			}
+		}
+	}
+	gains := false
+	for _, k := range composed.Kinds {
+		if !slices.Contains(k.Capabilities, "verified-sender") {
+			continue
+		}
+		held := false
+		for _, h := range had {
+			if h.Seat == k.Seat && h.Kind == k.Kind && h.Module == k.Module && h.Node == k.Node &&
+				slices.Contains(h.Capabilities, "verified-sender") {
+				held = true
+			}
+		}
+		gains = gains || !held
+	}
+	if !gains {
+		return issued, false, nil
+	}
+	if traffic == nil {
+		traffic = map[string]json.RawMessage{}
+	}
+	raw, err := json.Marshal(composed.Kinds)
+	if err != nil {
+		return nil, false, err
+	}
+	traffic["kinds"] = raw
+	if fields["seat-traffic"], err = json.Marshal(traffic); err != nil {
+		return nil, false, err
+	}
+	body, err := json.Marshal(fields)
+	return body, true, err
+}
+
+// reissueVerifiedSenders issues again each membership that lacks a verified sender the records now give it:
+// read as last issued, its kinds replaced, published. One never issued is left to its own send. Failures are
+// said and passed over; the next self-check tries again.
+func reissueVerifiedSenders(r broker.Records, where broker.Placements,
+	last func(node, module string) ([]byte, error), publish func(node, module string, body []byte) error,
+	say func(format string, args ...any)) int {
+	issued := 0
+	for node, declared := range r.Assigned {
+		for _, d := range declared {
+			composed := broker.MembershipFor(node, d, where).SeatTraffic
+			if composed == nil || !slices.ContainsFunc(composed.Kinds, func(k broker.KindHeld) bool {
+				return slices.Contains(k.Capabilities, "verified-sender")
+			}) {
+				continue
+			}
+			was, err := last(node, d.Module)
+			if err != nil {
+				if !errors.Is(err, nats.ErrMsgNotFound) {
+					say("  %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
+				}
+				continue
+			}
+			body, changed, err := withVerifiedSenders(was, composed)
+			if err != nil {
+				say("  %s on %s: its membership could not be read to give its verified sender back: %v", d.Module, node, err)
+				continue
+			}
+			if !changed {
+				continue
+			}
+			if err := publish(node, d.Module, body); err != nil {
+				say("  %s on %s: its verified sender could not be issued again: %v", d.Module, node, err)
+				continue
+			}
+			say("  %s on %s: issued its membership again, with the verified sender its machines now pass for", d.Module, node)
+			issued++
+		}
+	}
+	return issued
+}
+
+// rootVerdictsOf is each machine's verdict reason, for saying why one was not free.
+func rootVerdictsOf(verdicts []rootVerdict) (free map[string]bool, whys map[string]string) {
+	free, whys = map[string]bool{}, map[string]string{}
+	for _, v := range verdicts {
+		if v.Free {
+			free[v.Machine] = true
+		}
+		whys[v.Machine] = strings.TrimSpace(v.Why)
+	}
+	return free, whys
+}
+
+// giveVerifiedSendersBack is the self-check's step: every machine judged, the memberships that lost a verified
+// sender to a failed judgement issued again. Never fails the probe: what could not be done is said.
+func giveVerifiedSendersBack(ctx context.Context, d *doctor) {
+	if d.js == nil || d.open == nil || d.open.inventory == nil {
+		return
+	}
+	records, err := d.open.inventory.BusRecords(ctx)
+	if err != nil {
+		fmt.Printf("  the memberships could not be read to give a verified sender back: %v\n", err)
+		return
+	}
+	// Judged for every machine the records name, as a push does, not only those the self-check judged.
+	r := newRootReader(ctx, d.open.inventory, d.js.Conn())
+	all := judgeRootFree(ctx, r, records.Nodes, time.Now())
+	records.RootFree, _ = rootVerdictsOf(all)
+	where := broker.PlacementsOf(records, records.Interchangeable)
+	bus := link.OverNATS{Conn: d.js.Conn(), JS: d.js.Context()}
+	reissueVerifiedSenders(records, where,
+		func(node, module string) ([]byte, error) {
+			m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module))
+			if err != nil {
+				return nil, err
+			}
+			return m.Data, nil
+		},
+		func(node, module string, body []byte) error { return bus.PublishMembership(ctx, node, module, body) },
+		func(format string, args ...any) { fmt.Printf(format+"\n", args...) })
+}