mesh-controller/cmd/mesh-controller/operator_wait_test.go

package main

import (
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386).

var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1",
	Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}},
	OwnSecrets: catalogue.OwnSecrets{
		"smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside},
		"broker":         {Path: "/s/broker"},
		"made":           {Path: "/s/made", Taken: catalogue.TakenAtStart},
		"licence":        {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
	}}

var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games",
	What: "the password of the source games"}

var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}

func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth {
	return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process",
		Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool",
		Reason: "the source games waits for its password", Waits: waits}
}

func factsGiven(given map[string]time.Time) operatorWaitFacts {
	return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest},
		given: map[string]map[string]time.Time{"mounts@workstation": given}}
}

// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there.
func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
	at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC)
	for _, c := range []struct {
		name string
		w    inventory.Wait
		f    operatorWaitFacts
		says string // "" when excused
	}{
		{"a member of an outside family, not given", passwordWait, factsGiven(nil), ""},
		{"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""},
		{"a declared setting", usernameWait, factsGiven(nil), ""},
		{"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"},
		{"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"},
		{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
		{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
		{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
		{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"},
		{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"},
		{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
		{"what was given cannot be read", passwordWait,
			operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
	} {
		err := checkWait("mounts", "workstation", c.w, c.f)
		switch {
		case c.says == "" && err != nil:
			t.Errorf("%s: refused: %v", c.name, err)
		case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
			t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
		}
	}
}

// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why.
func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) {
	given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()})
	got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given)
	if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") {
		t.Fatalf("a wait for a secret given: %+v", got[0])
	}
	got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil))
	if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") {
		t.Fatalf("a wait naming nothing: %+v", got[0])
	}
	got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil))
	if got[0].State != link.StateWaiting {
		t.Fatalf("two waits that check out: %+v", got[0])
	}
}

// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything
// else beside it is judged as before; and any build is excused, not only one that added something.
func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) {
	now := time.Now()
	since := now.Add(-time.Minute)
	healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
		Target: "mesh-mounts-apply.service", State: link.StateHealthy}
	f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false},
		health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now,
			Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}}
	h, why := moduleHealthWord("mounts", "workstation", since, f)
	if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") ||
		!strings.Contains(why, "nox secret ask workstation mounts smb-password-games") {
		t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why)
	}
	// A second resource unhealthy beside it: not yet, as before.
	down := healthy
	down.State, down.Reason = link.StateUnhealthy, "down"
	f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
		Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}}
	if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
		t.Fatalf("a resource down beside the wait reads %v %q", h, why)
	}
	// The password given and the module still saying it waits: not excused.
	f.waits = factsGiven(map[string]time.Time{"smb-password-games": now})
	f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now,
		Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}
	if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") {
		t.Fatalf("a wait for a secret given reads %v %q", h, why)
	}
	// Facts never read (no manifest): never excused.
	f.waits = operatorWaitFacts{}
	if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet {
		t.Fatalf("a wait nothing could check reads %v", h)
	}
}

func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) {
	t.Helper()
	open, err := k.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	for i, c := range open {
		if c.Key == needsOperatorKey("mounts", "workstation") {
			return &open[i], open
		}
	}
	return nil, open
}

// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the
// act; a statement without it clears it.
func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	ctx := t.Context()
	rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
	if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil {
		t.Fatal(err)
	}
	if got, _ := needsOperatorOpen(t, k); got != nil {
		t.Fatal("raised on one statement")
	}
	if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil {
		t.Fatal(err)
	}
	got, open := needsOperatorOpen(t, k)
	if got == nil {
		t.Fatalf("not raised on two statements: %+v", open)
	}
	for _, c := range open {
		if c.Kind == kindModuleUnhealthy {
			t.Fatalf("raised as a fault too: %+v", c)
		}
	}
	if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning {
		t.Fatalf("the condition: %+v", got)
	}
	if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") ||
		!strings.Contains(got.Explanation, "hidden prompt opens at the desk") {
		t.Fatalf("its needs do not name the act: %q", got.Needs)
	}
	if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") {
		t.Fatalf("its explanation: %q", got.Explanation)
	}
	if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") {
		t.Fatalf("its summary does not name the secret and the line: %q", got.Summary)
	}
	// Long open is still a warning: only the operator can end it.
	if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil {
		t.Fatal(err)
	}
	if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent {
		t.Fatalf("after two days: %+v", got)
	}
	// Given: the next statement does not say it, and it clears.
	if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
		t.Fatal(err)
	}
	if got, _ := needsOperatorOpen(t, k); got != nil {
		t.Fatal("not cleared once given")
	}
}

func TestASettingsWaitAsksForTheApproval(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	ctx := t.Context()
	rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}}
	for i := 1; i <= 2; i++ {
		if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
			t.Fatal(err)
		}
	}
	got, _ := needsOperatorOpen(t, k)
	if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") {
		t.Fatalf("the condition: %+v", got)
	}
}

// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator.
func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	ctx := t.Context()
	checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)},
		factsGiven(map[string]time.Time{"smb-password-games": time.Now()}))
	rs := map[string][]inventory.ResourceHealth{"mounts": checked}
	for i := 1; i <= 2; i++ {
		if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil {
			t.Fatal(err)
		}
	}
	got, open := needsOperatorOpen(t, k)
	if got != nil {
		t.Fatalf("a wait for a secret given raised needs-operator: %+v", got)
	}
	unhealthy := false
	for _, c := range open {
		unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation")
	}
	if !unhealthy {
		t.Fatalf("not raised as unhealthy: %+v", open)
	}
}

// A module that waited and is then broken says so when the wait clears, and the other way round.
func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	ctx := t.Context()
	waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}
	for i := 1; i <= 2; i++ {
		if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil {
			t.Fatal(err)
		}
	}
	broken := waitingResource()
	broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil
	for i := 3; i <= 4; i++ {
		if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}},
			map[string]int{"mounts": i}, time.Now()); err != nil {
			t.Fatal(err)
		}
	}
	got, open := needsOperatorOpen(t, k)
	if got != nil {
		t.Fatal("needs-operator still open after it became a fault")
	}
	found := false
	for _, c := range open {
		found = found || c.Key == moduleUnhealthyKey("mounts", "workstation")
	}
	if !found {
		t.Fatalf("the fault is not raised: %+v", open)
	}
}