Research 031 counted the repairs people made by hand: a push to unstick a plan waiting on a report, a controller restarted to make an object again, a plan closed, a consumer re-made from now. Each was the ordinary path taken again by someone who noticed. The healer registry makes each a registered response to one condition kind, with a budget, a settle and its event: - H1 sent-not-reported: ask the machine's node-engine to report again (mesh.node.<n>.ask.report); if it does not report what it was sent, send it again, never moving a build a policy or a plan holds back - H2 stalled: close a plan whose wait is superseded or finished - H3 holder-silent / consumer-lost: the send's own assertion of the bus's objects (issue 208's note) - H4 consumer-behind: consumer-reset, only for a consumer the stream table marks resettable (the controller's own events consumer) - H5 is the identity provider's own repair (ADR 0224 §5), registered only Success is the observation clearing the condition, never the healer; a spent budget hands the condition to the operator, urgent, with what was tried, and no healer touches it again. Every act is begun in the store before it is made (migration 0070), kept in the condition's tried as "healer Hn" and said as the seat event healer-acted; a heal is never a hand act. More than twelve acts in an hour stop every healer until an hour after the last, said urgently. Only the lease holder heals. S15 is live: a cause repaired by hand twice in a fortnight raises healer-wanted, naming the healer that was not enough where one exists. D6's far-behind finding has its own kind, consumer-behind. Nodes are granted the question; the controller's grant gains healer-acted (genesis lock in mesh-host). `healers` lists the registry, the acts and the brake; status counts the week's heals.
106 lines
4.1 KiB
Go
106 lines
4.1 KiB
Go
package conditions
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// **A healer's attempt is said and kept, and never clears the condition** (to-be 45 §7): the resolver
|
|
// becomes the healer, `tried` says what it did, and only an observation clears it.
|
|
func TestAHealersAttemptIsKeptAndClearsNothing(t *testing.T) {
|
|
k, _, told, c := keeper(t)
|
|
ctx := t.Context()
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held, open, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked ace to report", Outcome: "acted",
|
|
By: "healer H1"}, ResolverHealer("H1"))
|
|
if err != nil || !open {
|
|
t.Fatalf("tried: %v %v", open, err)
|
|
}
|
|
if held.Resolver != "healer:H1" || len(held.Tried) != 1 || held.Tried[0].By != "healer H1" || held.Tried[0].At.IsZero() {
|
|
t.Fatalf("after one attempt: %+v", held)
|
|
}
|
|
said := settled(t, told, 2)
|
|
if said[1].Event != EventChanged || said[1].Change != ChangeResolver || said[1].Was != ResolverSelf {
|
|
t.Errorf("the healer taking it is not said as a change of resolver: %+v", said[1])
|
|
}
|
|
// A second attempt by the same healer is kept and says nothing new.
|
|
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "sent again", Outcome: "acted", By: "healer H1"},
|
|
ResolverHealer("H1")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _, _ := k.Get(ctx, "machine.ace.silent"); len(got.Tried) != 2 {
|
|
t.Errorf("tried %+v", got.Tried)
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
if n := len(told.Said()); n != 2 {
|
|
t.Errorf("a second attempt said %d events in all, want 2", n)
|
|
}
|
|
// Nothing open: nothing recorded, and no error.
|
|
if _, open, err := k.Tried(ctx, "machine.nobody.silent", Attempt{What: "x"}, ResolverHealer("H1")); open || err != nil {
|
|
t.Errorf("an attempt on nothing open: %v %v", open, err)
|
|
}
|
|
c.pass(time.Minute)
|
|
}
|
|
|
|
// **A spent budget is the operator's, urgent, and the watchdog seeing it again does not undo that.**
|
|
func TestAnEscalationHoldsAgainstTheNextObservation(t *testing.T) {
|
|
k, _, told, _ := keeper(t)
|
|
ctx := t.Context()
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked", Outcome: "acted", By: "healer H1"},
|
|
ResolverHealer("H1")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held, _, err := k.Escalate(ctx, "machine.ace.silent", Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if held.Severity != Urgent || held.Resolver != ResolverOperator || !held.Escalated() {
|
|
t.Fatalf("escalated: %+v", held)
|
|
}
|
|
said := settled(t, told, 4)
|
|
if said[2].Change != ChangeSeverity || said[3].Change != ChangeResolver || said[3].Was != "healer:H1" {
|
|
t.Errorf("the escalation is not said as severity then resolver: %+v %+v", said[2], said[3])
|
|
}
|
|
again, err := k.Observe(ctx, silent("ace")) // a warning, as the watchdog says it
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if again.Severity != Urgent || again.Resolver != ResolverOperator {
|
|
t.Errorf("the next observation undid the escalation: %+v", again)
|
|
}
|
|
// A healer trying later does not take it back from the operator.
|
|
after, _, _ := k.Tried(ctx, "machine.ace.silent", Attempt{What: "x", By: "healer H1"}, ResolverHealer("H1"))
|
|
if after.Resolver != ResolverOperator {
|
|
t.Errorf("a later attempt took the condition back from the operator: %s", after.Resolver)
|
|
}
|
|
}
|
|
|
|
// **What was tried is carried into a reopening**: the same fault again within ten minutes is the
|
|
// same condition, and what the healers tried on it is still what they tried.
|
|
func TestAReopeningCarriesWhatWasTried(t *testing.T) {
|
|
k, _, _, c := keeper(t)
|
|
ctx := t.Context()
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked", By: "healer H1"}, ResolverHealer("H1")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c.pass(5 * time.Minute)
|
|
again, err := k.Observe(ctx, silent("ace"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if again.Count != 2 || len(again.Tried) != 1 || again.Tried[0].What != "asked" {
|
|
t.Errorf("reopened without what was tried: %+v", again)
|
|
}
|
|
}
|