218 lines
8.4 KiB
Go
218 lines
8.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
|
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
|
|
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
|
|
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
|
|
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
|
|
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
|
|
|
|
// raisedKinds is every condition kind the mesh raises, and what raises it.
|
|
func raisedKinds() map[string]string {
|
|
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
|
|
for _, r := range signalsTable {
|
|
for _, k := range kindsOf(r) {
|
|
out[k] = r.Row
|
|
}
|
|
}
|
|
for _, p := range probeRegistry {
|
|
out[p.Kind] = p.ID
|
|
for _, k := range p.Raises {
|
|
out[k] = p.ID
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
|
|
var inducedFailures = map[string]string{
|
|
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
|
|
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
|
|
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
|
|
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
|
|
}
|
|
|
|
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
|
|
kinds := raisedKinds()
|
|
// The induced failures are tests of the integration level (a bus), in the file beside this one (issue 895).
|
|
var source []byte
|
|
for _, name := range []string{"healers_test.go", "healers_integration_test.go"} {
|
|
more, err := os.ReadFile(name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source = append(source, more...)
|
|
}
|
|
seen := map[string]bool{}
|
|
answered := map[string]string{}
|
|
for _, r := range healerRegistry {
|
|
t.Run(r.ID, func(t *testing.T) {
|
|
if seen[r.ID] {
|
|
t.Fatalf("%s is in the registry twice", r.ID)
|
|
}
|
|
seen[r.ID] = true
|
|
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
|
|
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
|
|
}
|
|
for _, k := range r.Kinds {
|
|
if _, ok := kinds[k]; !ok {
|
|
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
|
|
}
|
|
if other, twice := answered[k]; twice {
|
|
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
|
|
}
|
|
answered[k] = r.ID
|
|
}
|
|
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
|
|
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
|
|
}
|
|
if r.Event == "" {
|
|
t.Errorf("%s says nothing when it acts", r.ID)
|
|
}
|
|
if r.ActsIn != actsInController {
|
|
if r.repair != nil || r.applies != nil {
|
|
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
|
|
}
|
|
return
|
|
}
|
|
if r.repair == nil || r.applies == nil {
|
|
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
|
|
}
|
|
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
|
|
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
|
|
}
|
|
if inducedFailures[r.ID] == "" {
|
|
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
|
|
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
|
|
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
|
|
}
|
|
})
|
|
}
|
|
for id := range inducedFailures {
|
|
if !seen[id] {
|
|
t.Errorf("an induced failure for %s, which the registry does not have", id)
|
|
}
|
|
}
|
|
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
|
|
t.Error("the mesh-wide brake holds nothing")
|
|
}
|
|
}
|
|
|
|
// heard keeps the healer-acted events said.
|
|
type heard struct {
|
|
mu sync.Mutex
|
|
acts []healerActed
|
|
}
|
|
|
|
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
|
if seat != conditions.Seat || event != link.KeyHealerActed {
|
|
return errors.New("said under the wrong seat or name: " + seat + " " + event)
|
|
}
|
|
var e healerActed
|
|
if err := json.Unmarshal(body, &e); err != nil {
|
|
return err
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.acts = append(h.acts, e)
|
|
return nil
|
|
}
|
|
|
|
func (h *heard) said() []healerActed {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
return append([]healerActed(nil), h.acts...)
|
|
}
|
|
|
|
// testClock is a moment a test moves by hand.
|
|
type testClock struct {
|
|
mu sync.Mutex
|
|
at time.Time
|
|
}
|
|
|
|
func (c *testClock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.at
|
|
}
|
|
|
|
func (c *testClock) pass(d time.Duration) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.at = c.at.Add(d)
|
|
}
|
|
|
|
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
|
|
// fake that fails the test unless the test gives it.
|
|
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
|
|
t.Helper()
|
|
if conditionsFrom == nil {
|
|
t.Fatal("the mesh has no condition store")
|
|
}
|
|
told, clock := &heard{}, &testClock{at: time.Now()}
|
|
// The store's gate, as the serving controller's is the lease's (stores.go).
|
|
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
|
h := &healing{open: open, keeper: conditionsFrom, teller: told,
|
|
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
|
|
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
|
|
declined: map[string]string{}}
|
|
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
|
|
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
|
|
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
|
|
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
|
|
return h, told, clock
|
|
}
|
|
|
|
// sentNotReported raises S2 for a machine, as the watchdog does.
|
|
func sentNotReported(t *testing.T, node string) string {
|
|
t.Helper()
|
|
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
|
|
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
|
|
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return o.Key()
|
|
}
|
|
|
|
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
|
|
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
|
|
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
|
|
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
|
|
|
|
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
|
|
|
|
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
|
|
// after the last.
|
|
|
|
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
|
|
// still to come: that one is its own signal's, not a healer's.
|
|
|
|
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
|
|
// and left — what a reset drops, nothing would catch up for it.
|
|
|
|
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
|
|
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
|
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
|
|
|
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
|
|
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
|
// receives what comes next: a reset that left the controller deaf would be the incident.
|
|
|
|
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|