package main import ( "context" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one // unhealthy database provider and three consumers failing their checks that need it — one condition, at the // provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is // healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised // on its own from the start. func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory k := conditionsFrom register(t, open, catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) consumers := []string{"shop", "wiki", "crm"} for _, c := range consumers { register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}}) } if _, err := inv.Assign(ctx, "anchor", "db"); err != nil { t.Fatal(err) } machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"} for _, c := range consumers { if _, err := inv.Assign(ctx, machineOf[c], c); err != nil { t.Fatal(err) } if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c, Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { t.Fatal(err) } } at := h0 say := func(machine string, rs ...link.ResourceHealth) { t.Helper() at = at.Add(time.Second) for i := range rs { rs[i].Since = at } if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { t.Fatal(err) } } dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy, Reason: "its command: the database refuses connections", Check: "exec"} dbUp := dbDown dbUp.State, dbUp.Reason = link.StateHealthy, "" failing := func(module string) link.ResourceHealth { return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} } fine := func(module string) link.ResourceHealth { r := failing(module) r.State, r.Reason = link.StateHealthy, "" return r } openKeys := func() []conditions.Condition { t.Helper() list, err := k.Open(ctx) if err != nil { t.Fatal(err) } return list } // Two looks of the provider down and its consumers failing, in either order on each machine. for look := 0; look < 2; look++ { say("laptop", failing("shop"), failing("wiki")) say("anchor", dbDown, failing("crm")) } raised := openKeys() if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" { var keys []string for _, c := range raised { keys = append(keys, c.Key) } t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys) } c := raised[0] if c.Severity != conditions.Urgent { t.Errorf("consumers wait on the provider and its condition is %s", c.Severity) } said := c.Evidence[0].Said for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} { if !strings.Contains(said, w) { t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said) } } // Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too. f, err := gatherGateFacts(ctx, open, "") if err != nil { t.Fatal(err) } f.open, f.openErr, f.judged = raised, nil, false for _, m := range consumers { h, why := moduleHealthWord(m, machineOf[m], h0, f) if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") { t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why) } } // And a whole judging past the bound puts nothing back: it waits. long := h0.Add(-time.Hour) for _, m := range []string{"anchor", "laptop"} { f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true} } gatherWas := gatherGateFacts defer func() { gatherGateFacts = gatherWas }() gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil } g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long} verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now()) if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") { t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last) } // The provider healthy again: a consumer still failing is now its own, at once (its streak stood). say("anchor", dbUp, fine("crm")) say("laptop", failing("shop"), fine("wiki")) var keys []string for _, c := range openKeys() { keys = append(keys, c.Key) } if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) { t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys) } } // A consumer failing while its provider is healthy is raised on its own. func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory register(t, open, catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { t.Fatal(err) } } if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { t.Fatal(err) } healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"} shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} for i := 1; i <= 2; i++ { at := h0.Add(time.Duration(i) * time.Minute) _ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at) if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil { t.Fatal(err) } } list, _ := conditionsFrom.Open(ctx) if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" { t.Fatalf("a consumer failing beside a healthy provider raised %v", list) } }