mesh-controller/cmd/mesh-controller/provider_hold_test.go

package main

import (
	"context"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one
// unhealthy database provider and three consumers failing their checks that need it — one condition, at the
// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is
// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised
// on its own from the start.
func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	k := conditionsFrom
	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
	consumers := []string{"shop", "wiki", "crm"}
	for _, c := range consumers {
		register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}})
	}
	if _, err := inv.Assign(ctx, "anchor", "db"); err != nil {
		t.Fatal(err)
	}
	machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"}
	for _, c := range consumers {
		if _, err := inv.Assign(ctx, machineOf[c], c); err != nil {
			t.Fatal(err)
		}
		if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c,
			Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
			t.Fatal(err)
		}
	}

	at := h0
	say := func(machine string, rs ...link.ResourceHealth) {
		t.Helper()
		at = at.Add(time.Second)
		for i := range rs {
			rs[i].Since = at
		}
		if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
			t.Fatal(err)
		}
	}
	dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy,
		Reason: "its command: the database refuses connections", Check: "exec"}
	dbUp := dbDown
	dbUp.State, dbUp.Reason = link.StateHealthy, ""
	failing := func(module string) link.ResourceHealth {
		return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
			State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
	}
	fine := func(module string) link.ResourceHealth {
		r := failing(module)
		r.State, r.Reason = link.StateHealthy, ""
		return r
	}
	openKeys := func() []conditions.Condition {
		t.Helper()
		list, err := k.Open(ctx)
		if err != nil {
			t.Fatal(err)
		}
		return list
	}

	// Two looks of the provider down and its consumers failing, in either order on each machine.
	for look := 0; look < 2; look++ {
		say("laptop", failing("shop"), failing("wiki"))
		say("anchor", dbDown, failing("crm"))
	}
	raised := openKeys()
	if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" {
		var keys []string
		for _, c := range raised {
			keys = append(keys, c.Key)
		}
		t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys)
	}
	c := raised[0]
	if c.Severity != conditions.Urgent {
		t.Errorf("consumers wait on the provider and its condition is %s", c.Severity)
	}
	said := c.Evidence[0].Said
	for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} {
		if !strings.Contains(said, w) {
			t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said)
		}
	}

	// Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too.
	f, err := gatherGateFacts(ctx, open, "")
	if err != nil {
		t.Fatal(err)
	}
	f.open, f.openErr, f.judged = raised, nil, false
	for _, m := range consumers {
		h, why := moduleHealthWord(m, machineOf[m], h0, f)
		if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") {
			t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why)
		}
	}
	// And a whole judging past the bound puts nothing back: it waits.
	long := h0.Add(-time.Hour)
	for _, m := range []string{"anchor", "laptop"} {
		f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true}
	}
	gatherWas := gatherGateFacts
	defer func() { gatherGateFacts = gatherWas }()
	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil }
	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long}
	verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now())
	if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") {
		t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last)
	}

	// The provider healthy again: a consumer still failing is now its own, at once (its streak stood).
	say("anchor", dbUp, fine("crm"))
	say("laptop", failing("shop"), fine("wiki"))
	var keys []string
	for _, c := range openKeys() {
		keys = append(keys, c.Key)
	}
	if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) {
		t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys)
	}
}

// A consumer failing while its provider is healthy is raised on its own.
func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
	register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
	for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
		if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
			t.Fatal(err)
		}
	}
	if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
		Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
		t.Fatal(err)
	}
	healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"}
	shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy,
		Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
	for i := 1; i <= 2; i++ {
		at := h0.Add(time.Duration(i) * time.Minute)
		_ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at)
		if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil {
			t.Fatal(err)
		}
	}
	list, _ := conditionsFrom.Open(ctx)
	if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" {
		t.Fatalf("a consumer failing beside a healthy provider raised %v", list)
	}
}