Say when the mesh is wrong: conditions, watchdogs, the bus's advisories, doctor (hq to-be 45 Phase 1)
Every one of the 48 core failures of research 031 was found by a person looking; the mesh's answers carried the fact for whoever asked and told nobody. - The condition store (to-be 45 §2): mesh-controller_conditions, one key per open condition, written by compare-and-set so a person's silence and the watchdogs never lose each other's word; every transition kept ninety days in mesh-controller_condition-history and said as the seat's events condition-raised / condition-changed / condition-cleared (the condition at the top level, with event, at, change, why, show), offered again while the bus is away. Raised and cleared by observation only; a clearing reopened within ten minutes is the same condition with its count up, its silence kept. Verbs: conditions, conditions show, conditions silence (a hand act, at most a week), conditions history. - ADR 0224's provider standing is the first kind, provider-failing, held by the provider's events; the provider_standing table is no longer read or written (left in place: dropping it is the operator's word). - status leads with the open conditions, urgent first, and says all well only with none open; conditions it cannot read are said and not well. - The signals table compiled in, one watchdog loop over it every 30s: S1 heartbeat (3 intervals, asleep machines excepted, control node urgent after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf, S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9 advisories, S10 self-check silent, S11 node tools silent, S13 stale refusals; S12, S14, S15 deferred with their reasons. A row that cannot see raises probe-failed and clears nothing. A test generated from the table suppresses each signal inside and past its bound. - The bus's advisories (maximum deliveries, a mesh consumer deleted) and the controller's own slow consumer and refused subjects, said in the mesh's words. - doctor: the probe registry D1-D10 (D5 deferred) and DW, every five minutes, each in thirty seconds; a probe that cannot run is never a pass. D1 validates with mesh-host's own validator. Every run ends with the doctor-heartbeat event mesh-watcher listens for. - The controller is granted its new buckets, events, the two advisories and $SRV.INFO; the node tools their tools-alive heartbeat. The streams and consumers the controller asserts and the ones D6/D7 expect are one derivation.
This commit is contained in:
jochen
@@ -0,0 +1,159 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store on the bus (to-be 45 §2, ADR 0201): the controller's two buckets, asserted at
|
||||
// its start like its calls and its hand-act log.
|
||||
|
||||
// OnTheBus opens the store and its history on a connection.
|
||||
func OnTheBus(ctx context.Context, conn *nats.Conn) (Backend, History, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
open, err := api.KeyValue(ctx, broker.ConditionsBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition store %s is not on the bus — the controller asserts it at "+
|
||||
"its start, so one older than this has not: %w", broker.ConditionsBucket, err)
|
||||
}
|
||||
history, err := api.KeyValue(ctx, broker.ConditionHistoryBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition history %s is not on the bus — the controller asserts it "+
|
||||
"at its start, so one older than this has not: %w", broker.ConditionHistoryBucket, err)
|
||||
}
|
||||
return busStore{open}, &busHistory{api: api, kv: history}, nil
|
||||
}
|
||||
|
||||
type busStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (b busStore) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||
e, err := b.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Entry{}, false, err
|
||||
}
|
||||
return Entry{Value: e.Value(), Revision: e.Revision()}, true, nil
|
||||
}
|
||||
|
||||
func (b busStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
_, err := b.kv.Create(ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
_, err := b.kv.Update(ctx, key, value, revision)
|
||||
return moved(err)
|
||||
}
|
||||
|
||||
func (b busStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
return moved(b.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||
}
|
||||
|
||||
// moved reads the server's refusal of a compare-and-set as what it is.
|
||||
func moved(err error) error {
|
||||
var apiErr *jetstream.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// All is every key, read through a watch that hands over each current value and then says it has.
|
||||
func (b busStore) All(ctx context.Context) (map[string]Entry, error) {
|
||||
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
out := map[string]Entry{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the condition store: %w", ctx.Err())
|
||||
case e := <-w.Updates():
|
||||
if e == nil {
|
||||
return out, nil
|
||||
}
|
||||
out[e.Key()] = Entry{Value: e.Value(), Revision: e.Revision()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// busHistory keeps each transition under a key of its time and a sequence, and reads them back
|
||||
// from a moment through the stream under the bucket — by time, so a read of the last ten minutes
|
||||
// does not read ninety days.
|
||||
type busHistory struct {
|
||||
api jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
seq atomic.Uint64
|
||||
}
|
||||
|
||||
func (h *busHistory) Append(ctx context.Context, e Event) error {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := strconv.FormatInt(e.At.UnixNano(), 10) + "-" + strconv.FormatUint(h.seq.Add(1), 10)
|
||||
_, err = h.kv.Put(ctx, key, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// historyQuiet is how long a read of the history waits for one more transition before it takes the
|
||||
// stream as read to its end; it answers at once while it holds something.
|
||||
const historyQuiet = 2 * time.Second
|
||||
|
||||
func (h *busHistory) Since(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
start := since
|
||||
consumer, err := h.api.OrderedConsumer(ctx, "KV_"+broker.ConditionHistoryBucket, jetstream.OrderedConsumerConfig{
|
||||
DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &start,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
var out []Event
|
||||
pending := info.NumPending
|
||||
for pending > 0 {
|
||||
msg, err := consumer.Next(jetstream.FetchMaxWait(historyQuiet))
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// Nothing more within the quiet wait: read to its end.
|
||||
break
|
||||
}
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
pending = meta.NumPending
|
||||
var e Event
|
||||
if len(msg.Data()) > 0 && json.Unmarshal(msg.Data(), &e) == nil && e.Key != "" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||
// history read back by time are claims about what the bus does.
|
||||
|
||||
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, history, err := OnTheBus(t.Context(), js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, store, history
|
||||
}
|
||||
|
||||
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
||||
// other's word: a stale revision is refused as moved.
|
||||
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
told := &Told{}
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
|
||||
again := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
defer again.Close(context.Background())
|
||||
open, err := again.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Observations != 2 {
|
||||
t.Fatalf("a new keeper read %+v", open)
|
||||
}
|
||||
e, _, err := store.Get(ctx, "machine.ace.silent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a write at a stale revision answered %v", err)
|
||||
}
|
||||
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
||||
t.Fatalf("creating an open condition answered %v", err)
|
||||
}
|
||||
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a delete at a stale revision answered %v", err)
|
||||
}
|
||||
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
// Raised again at once: the store takes a key whose last word was a delete.
|
||||
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
||||
if got.Count != 2 {
|
||||
t.Fatalf("raised again after its clearing as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
||||
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
||||
t.Fatalf("history %+v", all)
|
||||
}
|
||||
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Fatalf("history from the future: %+v %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
|
||||
//
|
||||
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
|
||||
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
|
||||
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
|
||||
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
|
||||
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
|
||||
// resolved — never by hand.
|
||||
//
|
||||
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
|
||||
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
|
||||
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
|
||||
// itself rather than overwriting what the other said.
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
// Urgent needs the operator now.
|
||||
Urgent Severity = "urgent"
|
||||
// Warning needs the operator when they can.
|
||||
Warning Severity = "warning"
|
||||
)
|
||||
|
||||
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||
const (
|
||||
ScopeMachine = "machine"
|
||||
ScopePlan = "plan"
|
||||
ScopeCall = "call"
|
||||
ScopeBuild = "build"
|
||||
ScopeMerge = "merge"
|
||||
ScopeProvider = "provider"
|
||||
ScopeSeat = "seat"
|
||||
ScopeBus = "bus"
|
||||
ScopeCore = "core"
|
||||
ScopeProbe = "probe"
|
||||
ScopeMesh = "mesh"
|
||||
)
|
||||
|
||||
// Scopes is every scope, in the order a person reads them.
|
||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
|
||||
|
||||
// Who resolves a condition.
|
||||
const (
|
||||
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||
ResolverSelf = "self"
|
||||
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||
ResolverOperator = "operator"
|
||||
// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
|
||||
ResolverAgent = "agent"
|
||||
)
|
||||
|
||||
// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
|
||||
func ResolverHealer(name string) string { return "healer:" + name }
|
||||
|
||||
// Subject is what the condition is about: its scope, its id within the scope, and the machine it
|
||||
// concerns when there is one.
|
||||
type Subject struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||
Also []string `json:"also,omitempty"`
|
||||
}
|
||||
|
||||
// Evidence is one observation, as it was said.
|
||||
type Evidence struct {
|
||||
At time.Time `json:"at"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Attempt is one healer's try at a condition (to-be 45 §7; written from Phase 3).
|
||||
type Attempt struct {
|
||||
At time.Time `json:"at"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"`
|
||||
}
|
||||
|
||||
// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
|
||||
// act; the condition stays open, and `status` still says it.
|
||||
type Silence struct {
|
||||
Until time.Time `json:"until"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||
const KeptEvidence = 10
|
||||
|
||||
// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
|
||||
const MaxSilence = 7 * 24 * time.Hour
|
||||
|
||||
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
|
||||
// count increased, rather than news (to-be 45 §2).
|
||||
const ReopenWithin = 10 * time.Minute
|
||||
|
||||
// Condition is one open condition, as the store keeps it and its events carry it.
|
||||
type Condition struct {
|
||||
Key string `json:"key"`
|
||||
// Kind is the condition kind: from the signals table, the probe registry or an event kind.
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words.
|
||||
Summary string `json:"summary"`
|
||||
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
Source string `json:"source"`
|
||||
// Raised is when it was first observed this time; LastObserved the newest observation.
|
||||
Raised time.Time `json:"raised"`
|
||||
LastObserved time.Time `json:"last-observed"`
|
||||
// Observations is how many times it was observed since raised.
|
||||
Observations int `json:"observations"`
|
||||
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||
Count int `json:"count"`
|
||||
Tried []Attempt `json:"tried,omitempty"`
|
||||
Resolver string `json:"resolver"`
|
||||
// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
|
||||
Silenced *Silence `json:"silenced"`
|
||||
// Epoch is the controller lease epoch that last wrote it. Zero until the lease exists (to-be 45
|
||||
// Phase 2): no controller holds an epoch yet, and a number invented here would be one nobody
|
||||
// could compare.
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// SilencedAt says whether a person's silence is in force at a moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||
}
|
||||
|
||||
// Show is the verb that shows more about a condition, as a message carries it.
|
||||
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }
|
||||
|
||||
// Observation is one watchdog, probe or event saying something is wrong now.
|
||||
type Observation struct {
|
||||
Scope string
|
||||
// ID is the thing within the scope; several tokens joined by dots where the thing is named by
|
||||
// several (a provider's module, its machine and the consumer).
|
||||
ID string
|
||||
// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
|
||||
// provider. Kind's own word when empty.
|
||||
Token string
|
||||
Kind string
|
||||
Machine string
|
||||
// Also are the other machines it concerns.
|
||||
Also []string
|
||||
Severity Severity
|
||||
Summary string
|
||||
// Said is this observation's evidence, in the mesh's words; Summary when empty.
|
||||
Said string
|
||||
Source string
|
||||
Resolver string
|
||||
}
|
||||
|
||||
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
|
||||
// again is the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return Key(o.Scope, o.ID, token)
|
||||
}
|
||||
|
||||
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
|
||||
// tokens, and a `*` or `>` would make one a wildcard.
|
||||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||
|
||||
// Key composes a condition's key from its parts, each token made safe for the bus: a character the
|
||||
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
|
||||
func Key(scope, id, token string) string {
|
||||
var parts []string
|
||||
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||
if p == "" {
|
||||
p = "_"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
|
||||
// knows, or no severity is one nobody could route.
|
||||
func (o Observation) check() error {
|
||||
known := false
|
||||
for _, s := range Scopes {
|
||||
if s == o.Scope {
|
||||
known = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
|
||||
case strings.TrimSpace(o.ID) == "":
|
||||
return fmt.Errorf("a %s condition names what it is about", o.Scope)
|
||||
case strings.TrimSpace(o.Kind) == "":
|
||||
return fmt.Errorf("the condition %s has no kind", o.Key())
|
||||
case o.Severity != Urgent && o.Severity != Warning:
|
||||
return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
|
||||
case strings.TrimSpace(o.Summary) == "":
|
||||
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||
case strings.TrimSpace(o.Source) == "":
|
||||
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
|
||||
func Order(list []Condition) {
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
if list[i].Severity != list[j].Severity {
|
||||
return list[i].Severity == Urgent
|
||||
}
|
||||
if !list[i].Raised.Equal(list[j].Raised) {
|
||||
return list[i].Raised.Before(list[j].Raised)
|
||||
}
|
||||
return list[i].Key < list[j].Key
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package conditions
|
||||
|
||||
import "time"
|
||||
|
||||
// The events a condition's life emits (to-be 45 §2), as the mesh-controller seat's own: published on
|
||||
// `mesh.seat.mesh-controller.event.<name>`, on the events stream, so a consumer that was away catches
|
||||
// up. **This is a contract**: the operator-channel's holder is written against these names and the
|
||||
// shape of Event, and the controller learns nothing about telling.
|
||||
const (
|
||||
// EventRaised: a condition was raised — new, or the same fault again within ReopenWithin of its
|
||||
// clearing (Change says which). A reopened condition is not news: its key is the one the
|
||||
// first message was about.
|
||||
EventRaised = "condition-raised"
|
||||
// EventChanged: its severity, its resolver or its silence changed. Not every observation: a
|
||||
// condition observed again is written, and says nothing.
|
||||
EventChanged = "condition-changed"
|
||||
// EventCleared: an observation says it is resolved. The condition is removed from the store and
|
||||
// the transition kept in its history.
|
||||
EventCleared = "condition-cleared"
|
||||
)
|
||||
|
||||
// Events is every event a condition's life emits.
|
||||
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||
|
||||
// HeartbeatEvent is the self-check's heartbeat (to-be 45 §4, S10), said under the same seat at the end
|
||||
// of every run: `{run, at, interval-seconds, counts: {passed, failed, failed-to-run, deferred}, probes,
|
||||
// controller, why}`. mesh-watcher, on a machine that is not the control node, listens for it.
|
||||
const HeartbeatEvent = "doctor-heartbeat"
|
||||
|
||||
// What changed, as an event's Change and a history entry's says it.
|
||||
const (
|
||||
ChangeRaised = "raised"
|
||||
ChangeReopened = "reopened"
|
||||
ChangeSeverity = "severity"
|
||||
ChangeResolver = "resolver"
|
||||
ChangeSilenced = "silenced"
|
||||
ChangeUnsilenced = "silence-ended"
|
||||
ChangeCleared = "cleared"
|
||||
)
|
||||
|
||||
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||
// itself, at the top level** — key, kind, subject, severity, summary, source, raised, last-observed,
|
||||
// observations, resolver, silenced (null when not), epoch, and the evidence — with what happened to
|
||||
// it beside. One object a consumer reads the same way whichever of the three it is.
|
||||
type Event struct {
|
||||
// Condition is the condition after the transition — as it was last held, for a clearing.
|
||||
Condition
|
||||
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||
Event string `json:"event"`
|
||||
// At is when the transition happened.
|
||||
At time.Time `json:"at"`
|
||||
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, cleared.
|
||||
Change string `json:"change"`
|
||||
// Was is the value before, for a severity or resolver change.
|
||||
Was string `json:"was,omitempty"`
|
||||
// Why says why it cleared, or why it was silenced.
|
||||
Why string `json:"why,omitempty"`
|
||||
// Cleared is when it cleared, on a clearing.
|
||||
Cleared *time.Time `json:"cleared,omitempty"`
|
||||
// Show is the verb that shows more.
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// eventFor is the event a change is said under.
|
||||
func eventFor(change string) string {
|
||||
switch change {
|
||||
case ChangeRaised, ChangeReopened:
|
||||
return EventRaised
|
||||
case ChangeCleared:
|
||||
return EventCleared
|
||||
}
|
||||
return EventChanged
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
|
||||
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
|
||||
type InMemory struct {
|
||||
mu sync.Mutex
|
||||
values map[string]Entry
|
||||
revision uint64
|
||||
events []Event
|
||||
// Fail, when set, is what every read and write answers: a store that is away.
|
||||
Fail error
|
||||
}
|
||||
|
||||
// NewInMemory is an empty store.
|
||||
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
|
||||
|
||||
func (m *InMemory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return Entry{}, false, m.Fail
|
||||
}
|
||||
e, ok := m.values[key]
|
||||
return e, ok, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if _, ok := m.values[key]; ok {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) All(context.Context) (map[string]Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
out := make(map[string]Entry, len(m.values))
|
||||
for k, v := range m.values {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Append(_ context.Context, e Event) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
m.events = append(m.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Since(_ context.Context, since time.Time) ([]Event, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range m.events {
|
||||
if !e.At.Before(since) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Told is a teller that remembers what it was told, for tests.
|
||||
type Told struct {
|
||||
mu sync.Mutex
|
||||
Events []Event
|
||||
Names []string
|
||||
Fail error
|
||||
}
|
||||
|
||||
func (t *Told) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.Fail != nil {
|
||||
return t.Fail
|
||||
}
|
||||
if seat != Seat {
|
||||
return errors.New("told under the wrong seat: " + seat)
|
||||
}
|
||||
var e Event
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
t.Events = append(t.Events, e)
|
||||
t.Names = append(t.Names, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Said is a copy of what was told so far.
|
||||
func (t *Told) Said() []Event {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return append([]Event(nil), t.Events...)
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
|
||||
type Backend interface {
|
||||
// Get is one key's value and revision; false when it holds none.
|
||||
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||
// Create writes a key that holds nothing, and fails with ErrMoved when it holds something.
|
||||
Create(ctx context.Context, key string, value []byte) error
|
||||
// Update writes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||
// Delete removes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Delete(ctx context.Context, key string, revision uint64) error
|
||||
// All is every key's value. An error is an error: never an empty store (ADR 0227 rule 4).
|
||||
All(ctx context.Context) (map[string]Entry, error)
|
||||
}
|
||||
|
||||
// Entry is one key's value, at a revision.
|
||||
type Entry struct {
|
||||
Value []byte
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||
var ErrMoved = errors.New("the condition was written by somebody else since it was read")
|
||||
|
||||
// History keeps every transition (to-be 45 §2): appended, read back from a moment.
|
||||
type History interface {
|
||||
Append(ctx context.Context, e Event) error
|
||||
// Since is every transition from a moment, oldest first.
|
||||
Since(ctx context.Context, since time.Time) ([]Event, error)
|
||||
}
|
||||
|
||||
// Teller says a transition on the bus, as the mesh-controller seat's event. The link's bus is one.
|
||||
type Teller interface {
|
||||
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||
}
|
||||
|
||||
// Seat is the role the events are said under (novox/hq ADR 0134): the control plane's.
|
||||
const Seat = "mesh-controller"
|
||||
|
||||
// Keeper raises, observes, silences and clears conditions, and says each transition.
|
||||
type Keeper struct {
|
||||
store Backend
|
||||
history History
|
||||
teller Teller
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
changed func()
|
||||
|
||||
mu sync.Mutex
|
||||
// cleared is when each recently cleared condition cleared and how often it had been raised, so
|
||||
// one raised again within ReopenWithin is the same one again.
|
||||
cleared map[string]clearing
|
||||
|
||||
// out is the transitions still to be said and kept, in order: said by one goroutine, so a
|
||||
// condition's events arrive in the order they happened, and offered again while the bus is away.
|
||||
out chan Event
|
||||
drained chan struct{}
|
||||
closing sync.Once
|
||||
// Unsaid counts the transitions given up on, for the self-check to say.
|
||||
unsaid int
|
||||
}
|
||||
|
||||
type clearing struct {
|
||||
at time.Time
|
||||
count int
|
||||
silenced *Silence
|
||||
}
|
||||
|
||||
// Options are what a Keeper is made with.
|
||||
type Options struct {
|
||||
Store Backend
|
||||
History History
|
||||
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
|
||||
Teller Teller
|
||||
Now func() time.Time
|
||||
// Say is where a transition that could not be said or kept is said instead.
|
||||
Say func(format string, args ...any)
|
||||
// Changed is told of every transition, at once — for `status`, which leads with what is open.
|
||||
Changed func()
|
||||
}
|
||||
|
||||
// TellFor is how long one transition is offered to the bus before it is said lost.
|
||||
var TellFor = 10 * time.Minute
|
||||
|
||||
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
|
||||
// that cleared just before this controller started and is raised again now is a reopening.
|
||||
func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
|
||||
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
|
||||
if k.now == nil {
|
||||
k.now = time.Now
|
||||
}
|
||||
if k.say == nil {
|
||||
k.say = func(string, ...any) {}
|
||||
}
|
||||
if k.history != nil {
|
||||
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||
for _, e := range recent {
|
||||
if e.Change == ChangeCleared {
|
||||
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
k.say("what cleared lately could not be read from the condition history, so a condition "+
|
||||
"raised again now is said as new rather than reopened: %v", err)
|
||||
}
|
||||
}
|
||||
go k.telling()
|
||||
return k
|
||||
}
|
||||
|
||||
// Close says what is still to be said, waiting at most until ctx ends.
|
||||
func (k *Keeper) Close(ctx context.Context) {
|
||||
k.closing.Do(func() { close(k.out) })
|
||||
select {
|
||||
case <-k.drained:
|
||||
case <-ctx.Done():
|
||||
k.say("%d condition transition(s) were not yet said when this process ended", len(k.out))
|
||||
}
|
||||
}
|
||||
|
||||
// Unsaid is how many transitions were given up on since this keeper started.
|
||||
func (k *Keeper) Unsaid() int {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.unsaid
|
||||
}
|
||||
|
||||
// tries bounds one compare-and-set: two writers rarely race more than once.
|
||||
const tries = 8
|
||||
|
||||
// Observe records one observation: raises the condition if it is not open, and otherwise adds the
|
||||
// evidence. Says a raising, a reopening, and a change of severity or resolver; an observation that
|
||||
// changes neither is written and said nowhere.
|
||||
func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) {
|
||||
if err := o.check(); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
key := o.Key()
|
||||
for i := 0; i < tries; i++ {
|
||||
now := k.now().UTC()
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
k.mu.Lock()
|
||||
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||
c.Count, change = before.count+1, ChangeReopened
|
||||
// A silence a person gave the condition before it cleared still holds: they said
|
||||
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||
c.Silenced = before.silenced
|
||||
}
|
||||
}
|
||||
k.mu.Unlock()
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Create(ctx, key, body); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
|
||||
}
|
||||
k.mu.Lock()
|
||||
delete(k.cleared, key)
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: change})
|
||||
return c, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
var changes []Event
|
||||
if o.Severity != c.Severity {
|
||||
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
|
||||
c.Severity = o.Severity
|
||||
}
|
||||
if r := orSelf(o.Resolver); o.Resolver != "" && r != c.Resolver {
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
|
||||
c.Resolver = r
|
||||
}
|
||||
c.Summary, c.Source, c.LastObserved = o.Summary, o.Source, now
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
if len(o.Also) > 0 {
|
||||
c.Subject.Also = o.Also
|
||||
}
|
||||
c.Observations++
|
||||
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||
if len(c.Evidence) > KeptEvidence {
|
||||
c.Evidence = c.Evidence[:KeptEvidence]
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
|
||||
}
|
||||
for _, e := range changes {
|
||||
e.At, e.Condition = now, c
|
||||
k.tell(e)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return false, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
// Unreadable is not resolved: kept, and said, rather than removed unread.
|
||||
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
|
||||
}
|
||||
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
k.mu.Lock()
|
||||
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced}
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Reconcile is one source's whole observation: every condition it observes is observed, and every
|
||||
// condition it raised before and no longer observes is cleared — the observation says it is
|
||||
// resolved. A source that could not observe must not call this: an empty observation clears all it
|
||||
// raised, which is exactly the fault of saying "none" for "I could not tell" (ADR 0227 rule 4).
|
||||
func (k *Keeper) Reconcile(ctx context.Context, source string, observed []Observation) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var problems []string
|
||||
for _, o := range observed {
|
||||
o.Source = source
|
||||
seen[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range all {
|
||||
if c.Source != source || seen[c.Key] {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, source+" no longer observes it"); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Silence stops a condition's messages for a while, with a reason, by somebody (to-be 45 §2). The
|
||||
// condition stays open and `status` still says it; recording the act in the hand-act log is the
|
||||
// caller's, which knows who acted.
|
||||
func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, why string) (Condition, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return Condition{}, errors.New("a silence says why: --why <text>")
|
||||
}
|
||||
if d <= 0 || d > MaxSilence {
|
||||
return Condition{}, fmt.Errorf("a condition is silenced for a while, at most %s — not %s", MaxSilence, d)
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return Condition{}, fmt.Errorf("no condition %s is open — `conditions` lists them", key)
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("silencing the condition %s: %w", key, err)
|
||||
}
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeSilenced, Why: c.Silenced.Why})
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this silence; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// EndSilences ends every silence that has run out, and says each: the condition is still open, and
|
||||
// its messages start again.
|
||||
func (k *Keeper) EndSilences(ctx context.Context) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := k.now().UTC()
|
||||
for _, c := range all {
|
||||
if c.Silenced == nil || now.Before(c.Silenced.Until) {
|
||||
continue
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, c.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
var held Condition
|
||||
if err := json.Unmarshal(entry.Value, &held); err != nil {
|
||||
return fmt.Errorf("the condition %s on the bus cannot be read: %w", c.Key, err)
|
||||
}
|
||||
if held.Silenced == nil || now.Before(held.Silenced.Until) {
|
||||
break
|
||||
}
|
||||
was := held.Silenced.Why
|
||||
held.Silenced = nil
|
||||
body, err := json.Marshal(held)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.store.Update(ctx, c.Key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
k.tell(Event{Condition: held, At: now, Change: ChangeUnsilenced, Why: was})
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open is every open condition, urgent first and then oldest first.
|
||||
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
|
||||
return Read(ctx, k.store)
|
||||
}
|
||||
|
||||
// Get is one open condition.
|
||||
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
|
||||
return ReadOne(ctx, k.store, key)
|
||||
}
|
||||
|
||||
// HistorySince is every transition from a moment, oldest first.
|
||||
func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
if k.history == nil {
|
||||
return nil, errors.New("this keeper has no history to read")
|
||||
}
|
||||
return k.history.Since(ctx, since)
|
||||
}
|
||||
|
||||
// Read is every open condition in a store, in the order status says them. A value that cannot be
|
||||
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
|
||||
func Read(ctx context.Context, store Backend) ([]Condition, error) {
|
||||
all, err := store.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||
}
|
||||
out := make([]Condition, 0, len(all))
|
||||
for key, e := range all {
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return nil, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
Order(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadOne is one open condition from a store.
|
||||
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
|
||||
e, found, err := store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return Condition{}, found, err
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
|
||||
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
|
||||
func (k *Keeper) tell(e Event) {
|
||||
e.Event = eventFor(e.Change)
|
||||
e.Show = e.Condition.Show()
|
||||
if k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
defer func() {
|
||||
// A keeper closed while a write was in flight: said, not a panic.
|
||||
if recover() != nil {
|
||||
k.lost(e, errors.New("the keeper was closed"))
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case k.out <- e:
|
||||
default:
|
||||
k.lost(e, errors.New("too many transitions are waiting to be said"))
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) lost(e Event, err error) {
|
||||
k.mu.Lock()
|
||||
k.unsaid++
|
||||
k.mu.Unlock()
|
||||
k.say("the condition %s was %s and that could NOT be said or kept: %v", e.Key, e.Change, err)
|
||||
}
|
||||
|
||||
// telling keeps and says every transition in order, offering each again while the bus is away.
|
||||
func (k *Keeper) telling() {
|
||||
defer close(k.drained)
|
||||
for e := range k.out {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
k.lost(e, err)
|
||||
continue
|
||||
}
|
||||
deadline := time.Now().Add(TellFor)
|
||||
wait := 200 * time.Millisecond
|
||||
kept, said := k.history == nil, k.teller == nil
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if !kept {
|
||||
kept = k.history.Append(ctx, e) == nil
|
||||
}
|
||||
if !said {
|
||||
said = k.teller.PublishSeatEvent(ctx, Seat, e.Event, body) == nil
|
||||
}
|
||||
cancel()
|
||||
if kept && said {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
what := "said"
|
||||
if !kept {
|
||||
what = "kept in the history"
|
||||
}
|
||||
k.lost(e, fmt.Errorf("not %s within %s", what, TellFor))
|
||||
break
|
||||
}
|
||||
time.Sleep(wait)
|
||||
wait = min(2*wait, 10*time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orSelf(resolver string) string {
|
||||
if resolver == "" {
|
||||
return ResolverSelf
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clock is a time a test moves by hand.
|
||||
type clock struct{ at time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.at }
|
||||
func (c *clock) pass(d time.Duration) { c.at = c.at.Add(d) }
|
||||
func newClock() *clock { return &clock{at: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||
func keeper(t *testing.T) (*Keeper, *InMemory, *Told, *clock) {
|
||||
t.Helper()
|
||||
store, told, c := NewInMemory(), &Told{}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now,
|
||||
Say: func(f string, a ...any) { t.Logf(f, a...) }})
|
||||
t.Cleanup(func() { k.Close(context.Background()) })
|
||||
return k, store, told, c
|
||||
}
|
||||
|
||||
// settled waits until the teller has been told n events.
|
||||
func settled(t *testing.T, told *Told, n int) []Event {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
said := told.Said()
|
||||
if len(said) >= n {
|
||||
return said
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("told %d event(s), want %d: %+v", len(said), n, said)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func silent(node string) Observation {
|
||||
return Observation{Scope: ScopeMachine, ID: node, Kind: "silent", Machine: node, Severity: Warning,
|
||||
Summary: node + " has not been heard from", Source: "S1"}
|
||||
}
|
||||
|
||||
// **A condition is raised once, observed many times, and said on the bus only when it changes**
|
||||
// (to-be 45 §2): an observation that changes nothing is written and said nowhere, or the operator's
|
||||
// channel would hear the same fault every thirty seconds.
|
||||
func TestAConditionIsSaidWhenItChangesNotWhenItIsSeenAgain(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
urgent := silent("ace")
|
||||
urgent.Severity = Urgent
|
||||
got, err := k.Observe(ctx, urgent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Key != "machine.ace.silent" || got.Observations != 4 || got.Count != 1 || got.Severity != Urgent {
|
||||
t.Fatalf("held %+v", got)
|
||||
}
|
||||
if len(got.Evidence) != 4 || !got.Evidence[0].At.Equal(c.at) {
|
||||
t.Fatalf("evidence is not newest first: %+v", got.Evidence)
|
||||
}
|
||||
said := settled(t, told, 2)
|
||||
if said[0].Event != EventRaised || said[0].Change != ChangeRaised || said[1].Event != EventChanged ||
|
||||
said[1].Change != ChangeSeverity || said[1].Was != string(Warning) {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := len(told.Said()); n != 2 {
|
||||
t.Fatalf("said %d events for one raising and one change", n)
|
||||
}
|
||||
for i, name := range told.Names {
|
||||
if name != told.Events[i].Event {
|
||||
t.Errorf("event %d published as %s and says it is %s", i, name, told.Events[i].Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Evidence is bounded**: a condition open for a week keeps its newest ten observations, not all.
|
||||
func TestEvidenceKeepsTheNewestTen(t *testing.T) {
|
||||
k, _, _, c := keeper(t)
|
||||
var got Condition
|
||||
for i := 0; i < 25; i++ {
|
||||
var err error
|
||||
if got, err = k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
if len(got.Evidence) != KeptEvidence || got.Observations != 25 {
|
||||
t.Fatalf("kept %d evidence of %d observations", len(got.Evidence), got.Observations)
|
||||
}
|
||||
}
|
||||
|
||||
// **Cleared and raised again within ten minutes is the same condition again** (to-be 45 §2): its
|
||||
// count goes up and it is said as reopened, not as news; a person's silence of it still holds.
|
||||
func TestRaisedAgainSoonAfterClearingReopens(t *testing.T) {
|
||||
k, store, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "the laptop is on the train"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
again, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Count != 2 || again.Silenced == nil {
|
||||
t.Fatalf("reopened as %+v", again)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
if said[3].Event != EventRaised || said[3].Change != ChangeReopened {
|
||||
t.Fatalf("the reopening was said as %+v", said[3])
|
||||
}
|
||||
// And from a new keeper — the controller restarted between — reading what cleared from history.
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settled(t, told, 5)
|
||||
k.Close(context.Background())
|
||||
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
|
||||
defer next.Close(context.Background())
|
||||
c.pass(time.Minute)
|
||||
third, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if third.Count != 3 {
|
||||
t.Fatalf("a controller restarted between cleared and raised said it as new: %+v", third)
|
||||
}
|
||||
// Past the window it is news.
|
||||
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(ReopenWithin + time.Minute)
|
||||
fourth, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fourth.Count != 1 || fourth.Silenced != nil {
|
||||
t.Fatalf("raised past the window as %+v", fourth)
|
||||
}
|
||||
}
|
||||
|
||||
// **A source's whole observation clears what it no longer observes, and only its own.** A watchdog
|
||||
// that stops seeing a fault says it is resolved; it does not clear what another raised.
|
||||
func TestReconcileClearsOnlyTheSourcesOwn(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
other := Observation{Scope: ScopeProbe, ID: "D3", Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||
Summary: "D3 did not answer", Source: "doctor"}
|
||||
if _, err := k.Observe(ctx, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("ace"), silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
if strings.Join(keys, ",") != "machine.g14.silent,probe.D3.failed" {
|
||||
t.Fatalf("open after the second observation: %v", keys)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
last := said[3]
|
||||
if last.Event != EventCleared || last.Key != "machine.ace.silent" || last.Why == "" {
|
||||
t.Fatalf("the clearing was said as %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// **A store that cannot be read is never an empty one** (ADR 0227 rule 4): reconciling against it
|
||||
// clears nothing and says why.
|
||||
func TestAnUnreadableStoreClearsNothing(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Fail = errors.New("the bus is away")
|
||||
if err := k.Reconcile(ctx, "S1", nil); err == nil {
|
||||
t.Fatal("reconciled against a store it could not read")
|
||||
}
|
||||
if _, err := k.Open(ctx); err == nil {
|
||||
t.Fatal("an unreadable store answered as read")
|
||||
}
|
||||
store.Fail = nil
|
||||
store.values["machine.g14.silent"] = Entry{Value: []byte("{not a condition"), Revision: 99}
|
||||
if _, err := k.Open(ctx); err == nil || !strings.Contains(err.Error(), "machine.g14.silent") {
|
||||
t.Fatalf("an unreadable condition was left out rather than said: %v", err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.g14.silent", "x"); err == nil || cleared {
|
||||
t.Fatal("an unreadable condition was cleared unread")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence is bounded, says why, and ends on its own** (to-be 45 §2): the condition stays open
|
||||
// through it, and its messages start again when it ends.
|
||||
func TestASilenceIsBoundedAndEnds(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", 8*24*time.Hour, "jochen", "away"); err == nil {
|
||||
t.Fatal("silenced for longer than a week")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", " "); err == nil {
|
||||
t.Fatal("silenced without a reason")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.nothing.silent", time.Hour, "jochen", "x"); err == nil {
|
||||
t.Fatal("silenced a condition that is not open")
|
||||
}
|
||||
held, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "on the train")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !held.SilencedAt(c.at) || held.Silenced.By != "jochen" {
|
||||
t.Fatalf("silenced as %+v", held.Silenced)
|
||||
}
|
||||
c.pass(30 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(31 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if got.Silenced != nil {
|
||||
t.Fatalf("a silence past its end still held: %+v", got.Silenced)
|
||||
}
|
||||
said := settled(t, told, 3)
|
||||
if said[1].Change != ChangeSilenced || said[2].Change != ChangeUnsilenced || said[2].Event != EventChanged {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **Two writers never lose each other's word.** The serving controller observes while a person's
|
||||
// command silences: the write that lost the compare-and-set reads again and redoes itself.
|
||||
func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
racing := &racingStore{InMemory: store, before: func() {
|
||||
// Another process silences between this keeper's read and its write.
|
||||
other := NewKeeper(ctx, Options{Store: store})
|
||||
defer other.Close(context.Background())
|
||||
if _, err := other.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "known"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}}
|
||||
k.store = racing
|
||||
got, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Silenced == nil || got.Observations != 2 {
|
||||
t.Fatalf("the observation overwrote the silence: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// racingStore lets another writer in once, between a read and the write after it.
|
||||
type racingStore struct {
|
||||
*InMemory
|
||||
before func()
|
||||
done bool
|
||||
}
|
||||
|
||||
func (r *racingStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
if !r.done {
|
||||
r.done = true
|
||||
r.before()
|
||||
}
|
||||
return r.InMemory.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **An observation that could not be routed is refused**, naming what it lacks.
|
||||
func TestAnObservationSaysWhatItIs(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
for _, o := range []Observation{
|
||||
{Scope: "elsewhere", ID: "x", Kind: "k", Severity: Warning, Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: "loud", Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Summary: "s"},
|
||||
} {
|
||||
if _, err := k.Observe(t.Context(), o); err == nil {
|
||||
t.Errorf("observed %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A key holds nothing the bus would refuse or read as a wildcard**, whatever the thing is called.
|
||||
func TestAKeyIsSafeForTheBus(t *testing.T) {
|
||||
if got := Key(ScopeProvider, "keycloak.novox.my app*", "failing"); got != "provider.keycloak.novox.my_app_.failing" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
if got := Key(ScopeBus, "EVENTS.>", "consumer-lost"); got != "bus.EVENTS._.consumer-lost" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The event's shape is a contract** (to-be 45 §2): the operator-channel's holder is written against
|
||||
// these field names. A rename here is a channel that reads nothing, so they are held still.
|
||||
func TestTheEventShapeIsTheContract(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settled(t, told, 1)
|
||||
body, err := json.Marshal(said[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var shape map[string]any
|
||||
if err := json.Unmarshal(body, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The condition at the top level, kebab-case, beside what happened to it.
|
||||
for _, field := range []string{"event", "at", "change", "show", "key", "kind", "subject", "severity",
|
||||
"summary", "evidence", "source", "raised", "last-observed", "observations", "count", "resolver",
|
||||
"silenced", "epoch"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the event carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
if shape["silenced"] != nil {
|
||||
t.Errorf("an unsilenced condition says silenced %v, not null", shape["silenced"])
|
||||
}
|
||||
subject, _ := shape["subject"].(map[string]any)
|
||||
if subject["scope"] != "machine" || subject["id"] != "ace" || subject["machine"] != "ace" {
|
||||
t.Errorf("subject %v", shape["subject"])
|
||||
}
|
||||
if said[0].Show != "mesh-controller.conditions key=machine.ace.silent" {
|
||||
t.Errorf("show is %q", said[0].Show)
|
||||
}
|
||||
}
|
||||
|
||||
// **A transition the bus will not take is offered again**, and said lost only after TellFor.
|
||||
func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
|
||||
store, told, c := NewInMemory(), &Told{Fail: errors.New("no responders")}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now})
|
||||
defer k.Close(context.Background())
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
told.mu.Lock()
|
||||
told.Fail = nil
|
||||
told.mu.Unlock()
|
||||
said := settled(t, told, 1)
|
||||
if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 {
|
||||
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user