The identity provider failed every consumer for a day and status called the mesh well (hq issue 179). The controller now follows every provider's provisioner.failing/recovered, keeps the newest failing word per provider, machine and consumer (migration 0065), and status, its JSON and node show name it until it recovers. Every module that receives contributions is granted the two events, so no manifest can forget them.
131 lines
4.7 KiB
Go
131 lines
4.7 KiB
Go
package inventory
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
)
|
|
|
|
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
|
|
// reads.
|
|
|
|
// The broker spells the events itself because it cannot import the catalogue; the two agree.
|
|
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
|
|
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
|
|
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
|
|
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
|
|
}
|
|
}
|
|
|
|
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
|
|
// events would have its announcement refused by the bus, and fail its consumers as silently as on
|
|
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
|
|
// nothing.
|
|
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
|
|
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
|
|
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
|
|
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
|
|
|
|
d := declaredFor(provider, nil)
|
|
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
|
|
if !slices.Contains(d.Emits, e) {
|
|
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
|
|
}
|
|
}
|
|
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
|
|
Module: "keycloak", Emits: d.Emits})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
|
|
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
|
|
}
|
|
|
|
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
|
|
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
|
|
}
|
|
// Declared by hand as well: said once.
|
|
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
|
|
n := 0
|
|
for _, e := range provider.EmitsAll() {
|
|
if e == catalogue.ProvisionerFailing {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("%v", provider.EmitsAll())
|
|
}
|
|
}
|
|
|
|
// And the controller may hear it from every provider, and only those two events.
|
|
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
|
|
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
|
|
if !slices.Contains(perms.Subscribe, want) {
|
|
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
|
|
}
|
|
}
|
|
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
|
|
t.Fatal("the controller hears every event in the mesh")
|
|
}
|
|
}
|
|
|
|
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
|
|
inv := ForTest(t)
|
|
ctx := t.Context()
|
|
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
|
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
|
|
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
|
|
Error: "401 invalid_grant", Since: since, Attempts: 60}
|
|
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Said again: one row, the newest word.
|
|
s.Attempts = 31000
|
|
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := inv.FailingProviders(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
|
|
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if got[0].Quiet(time.Now()) {
|
|
t.Fatal("a standing just said reads as quiet")
|
|
}
|
|
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
|
|
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
|
|
}
|
|
|
|
// The same consumer from another machine's provider is its own row.
|
|
other := s
|
|
other.ProviderNode = "laptop"
|
|
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cleared, err := inv.KeepStanding(ctx, false, s)
|
|
if err != nil || !cleared {
|
|
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
|
|
}
|
|
cleared, err = inv.KeepStanding(ctx, false, s)
|
|
if err != nil || cleared {
|
|
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
|
|
}
|
|
got, err = inv.FailingProviders(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0].ProviderNode != "laptop" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
}
|