mesh-host/internal/link/health_test.go

package link

import (
	"context"
	"encoding/json"
	"slices"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// A health statement is said on the link open now, as the machine's own word on its own subject, and
// not said — for the caller to say again — while no link is open (novox/hq ADR 0240, to-be 48 §4).

// sayingLink is a link that can say health.
type sayingLink struct {
	*quietLink
	said [][]byte
}

func (l *sayingLink) Health(_ context.Context, node string, body []byte) error {
	l.said = append(l.said, body)
	return nil
}

func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
	q := &Session{Membership: Membership{Node: "anchor"}}
	h := Health{Contract: LivenessContract, At: time.Now().UTC(),
		Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy, Reason: ReasonRestarting}}}
	if q.SayHealth(t.Context(), h) {
		t.Fatal("said with no link open")
	}
	l := &sayingLink{quietLink: newQuietLink()}
	q.Attach(t.Context(), l)
	if !q.SayHealth(t.Context(), h) || len(l.said) != 1 {
		t.Fatalf("not said on the open link: %d", len(l.said))
	}
	var got HealthSaid
	if err := json.Unmarshal(l.said[0], &got); err != nil {
		t.Fatal(err)
	}
	if got.Node != "anchor" || len(got.Health.Resources) != 1 || got.Health.Resources[0].State != StateUnhealthy {
		t.Fatalf("said %+v", got)
	}
	// A link that cannot say it is not an error: the next report carries the statement.
	q.Detach(l)
	q.Attach(t.Context(), newQuietLink())
	if q.SayHealth(t.Context(), h) {
		t.Fatal("said on a link that cannot")
	}
	if HealthSubject("anchor") != "mesh.control.anchor.health" {
		t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
	}

	// Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on
	// a change only, the streaks and the time of the look not being one, and read first after a restart.
	t.Run("the node's health report is written on a change only", func(t *testing.T) {
		since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC)
		look := func(streak int, writer string) Health {
			return Health{Contract: 7, At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{
				{Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}},
				Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}},
				Resources: []ResourceHealth{
					{Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy,
						Reason: "relogin needed: the session began before scanner", Since: since},
					{Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy,
						Reason: ReasonUsedAsFound + " owned by root", Since: since},
					{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since},
					{Module: "claude-code", Resource: "claude-code.agent", Kind: KindAccount, Account: "agent", Root: "never",
						State: StateHealthy, Since: since}}}
		}
		peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
		r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{})
		if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
			len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
			len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" ||
			len(r.Root) != 1 || r.Root[0].Account != "agent" || r.Root[0].State != StateHealthy || r.Contract != 7 {
			t.Fatalf("the report is %+v", r)
		}
		// The machines an unhealthy part points at, as the controller holds it there; none for the machine's own.
		machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"}
		for _, tc := range []struct {
			name   string
			toward []string
			state  string
			want   []string
		}{
			{"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}},
			{"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}},
			{"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil},
			{"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil},
			{"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil},
		} {
			h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}}
			got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets
			if !slices.Equal(got, tc.want) {
				t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want)
			}
		}
		store := &memHealth{}
		var last *NodeHealth
		clock := func() time.Time { return since }
		for i, step := range []struct {
			h     NodeHealth
			fresh bool // a restart: nothing written in this run yet
			wrote bool
		}{
			{h: r, wrote: true},
			{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})},
			{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true},
			{h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true},
			{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true},
		} {
			if step.fresh {
				last = nil
			}
			wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock)
			if err != nil || wrote != step.wrote {
				t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err)
			}
		}
		got, err := delivery.DecodeNodeHealth(store.values["anchor"])
		if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() {
			t.Fatalf("the key holds %+v, %v", got, err)
		}
		// The key's bytes are mesh-sdk's layout: decoded and encoded again by it, the same bytes.
		if again, err := delivery.EncodeNodeHealth(got); err != nil || string(again) != string(store.values["anchor"]) {
			t.Fatalf("the key's bytes are not the sdk's layout: %s\n%s (%v)", store.values["anchor"], again, err)
		}
		if store.puts != 3 {
			t.Fatalf("%d writes, want 3: one per change", store.puts)
		}
	})
}

// memHealth is a HealthStore in memory.
type memHealth struct {
	values map[string][]byte
	puts   int
}

func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil }
func (m *memHealth) Put(_ context.Context, key string, value []byte) error {
	if m.values == nil {
		m.values = map[string][]byte{}
	}
	m.values[key], m.puts = value, m.puts+1
	return nil
}

// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
// ADR 0240, to-be 48 §2), and asked on this machine's own instance (to-be 48 §3).
func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
	for reply, want := range map[string]bool{
		`{"result":{"healthy":true},"node":"anchor"}`:                       true,
		`{"result":{"healthy":false,"why":"the admin refused the secret"}}`: false,
		`{"result":{"status":"fine"}}`:                                      false,
		`{"error":"503 no responders"}`:                                     false,
		`{"result":"yes"}`:                                                  false,
	} {
		got, err := ReadToolAnswer([]byte(reply))
		if err != nil || got.Healthy != want {
			t.Errorf("%s read as %v (%v)", reply, got, err)
		}
	}
	if ToolSubject("keycloak", "keycloak_admin_health", "anchor") != "mesh.mod.keycloak.tool.keycloak_admin_health.anchor" {
		t.Errorf("asked on %s", ToolSubject("keycloak", "keycloak_admin_health", "anchor"))
	}
}

// A declared tool check is asked on the link the node-engine actually holds — the NATS link — and not
// refused as if none were open (2026-10-08: every tool check read "no link to the bus is open" while
// the link was up, because the NATS link could not Ask). With no link open, and with a link that cannot
// ask, the check fails and says which.
func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
	q := &Session{Membership: Membership{Node: "anchor"}}
	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
		t.Fatalf("with no link open: %v", err)
	}

	quiet := newQuietLink()
	q.Attach(t.Context(), quiet)
	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
		t.Fatalf("on a link that cannot ask: %v", err)
	}
	q.Detach(quiet)

	// The link Open returns, with no connection under it: the question must reach its Ask (and fail
	// there), never stop at the queue's assertion.
	held := &natsLink{node: "anchor"}
	q.Attach(t.Context(), held)
	_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
	if err == nil {
		t.Fatal("asked on no connection and was answered")
	}
	if strings.Contains(err.Error(), "no link") || strings.Contains(err.Error(), "cannot ask a tool") {
		t.Fatalf("the NATS link was not asked: %v", err)
	}
}

// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
// healthy, so nothing reading `healthy` alone takes it for a pass.
func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
	got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
		`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
	if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
		t.Fatalf("read as %+v (%v)", got, err)
	}
	h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
	b, _ := json.Marshal(h)
	if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
		t.Fatalf("said on the wire as %s", b)
	}
}