Every one of the 48 core failures of research 031 was found by a person looking; the mesh's answers carried the fact for whoever asked and told nobody. - The condition store (to-be 45 §2): mesh-controller_conditions, one key per open condition, written by compare-and-set so a person's silence and the watchdogs never lose each other's word; every transition kept ninety days in mesh-controller_condition-history and said as the seat's events condition-raised / condition-changed / condition-cleared (the condition at the top level, with event, at, change, why, show), offered again while the bus is away. Raised and cleared by observation only; a clearing reopened within ten minutes is the same condition with its count up, its silence kept. Verbs: conditions, conditions show, conditions silence (a hand act, at most a week), conditions history. - ADR 0224's provider standing is the first kind, provider-failing, held by the provider's events; the provider_standing table is no longer read or written (left in place: dropping it is the operator's word). - status leads with the open conditions, urgent first, and says all well only with none open; conditions it cannot read are said and not well. - The signals table compiled in, one watchdog loop over it every 30s: S1 heartbeat (3 intervals, asleep machines excepted, control node urgent after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf, S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9 advisories, S10 self-check silent, S11 node tools silent, S13 stale refusals; S12, S14, S15 deferred with their reasons. A row that cannot see raises probe-failed and clears nothing. A test generated from the table suppresses each signal inside and past its bound. - The bus's advisories (maximum deliveries, a mesh consumer deleted) and the controller's own slow consumer and refused subjects, said in the mesh's words. - doctor: the probe registry D1-D10 (D5 deferred) and DW, every five minutes, each in thirty seconds; a probe that cannot run is never a pass. D1 validates with mesh-host's own validator. Every run ends with the doctor-heartbeat event mesh-watcher listens for. - The controller is granted its new buckets, events, the two advisories and $SRV.INFO; the node tools their tools-alive heartbeat. The streams and consumers the controller asserts and the ones D6/D7 expect are one derivation.
118 lines
3.5 KiB
Go
118 lines
3.5 KiB
Go
package conditions
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
)
|
|
|
|
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
|
// history read back by time are claims about what the bus does.
|
|
|
|
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
|
t.Helper()
|
|
url := os.Getenv("MESH_TEST_NATS")
|
|
if url == "" {
|
|
t.Skip("MESH_TEST_NATS unset")
|
|
}
|
|
js, err := broker.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(js.Close)
|
|
api, err := jetstream.New(js.Conn())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
|
if err := js.EnsureControllerBuckets(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, history, err := OnTheBus(t.Context(), js.Conn())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return js, store, history
|
|
}
|
|
|
|
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
|
// other's word: a stale revision is refused as moved.
|
|
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
|
_, store, history := busStoreForTest(t)
|
|
ctx := t.Context()
|
|
told := &Told{}
|
|
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k.Close(context.Background())
|
|
|
|
again := NewKeeper(ctx, Options{Store: store, History: history})
|
|
defer again.Close(context.Background())
|
|
open, err := again.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(open) != 1 || open[0].Observations != 2 {
|
|
t.Fatalf("a new keeper read %+v", open)
|
|
}
|
|
e, _, err := store.Get(ctx, "machine.ace.silent")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
|
t.Fatalf("a write at a stale revision answered %v", err)
|
|
}
|
|
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
|
t.Fatalf("creating an open condition answered %v", err)
|
|
}
|
|
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
|
t.Fatalf("a delete at a stale revision answered %v", err)
|
|
}
|
|
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
|
t.Fatalf("cleared %v: %v", cleared, err)
|
|
}
|
|
// Raised again at once: the store takes a key whose last word was a delete.
|
|
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
|
if got.Count != 2 {
|
|
t.Fatalf("raised again after its clearing as %+v", got)
|
|
}
|
|
}
|
|
|
|
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
|
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
|
_, store, history := busStoreForTest(t)
|
|
ctx := t.Context()
|
|
k := NewKeeper(ctx, Options{Store: store, History: history})
|
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k.Close(context.Background())
|
|
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
|
t.Fatalf("history %+v", all)
|
|
}
|
|
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
|
if err != nil || len(none) != 0 {
|
|
t.Fatalf("history from the future: %+v %v", none, err)
|
|
}
|
|
}
|