Keep a condition's looks in the store, so a look that changes nothing writes no bucket (issue 420)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

Every look rewrote the condition's value in the conditions bucket, because the
value carried the count of looks, the last look and the newest evidence: 131 647
writes in four days for twelve open conditions, against ADR 0289's rule that a
bucket is written only when what it describes changed. The looks now live in the
controller's store (migration 0095, condition_looks), the bucket holds what a
condition says and is written only when that changes, and the conditions verb and
the condition events put the looks back.
This commit is contained in:
jochen
2026-10-12 12:11:13 +02:00
parent 22a2f73f7b
commit dcdafd37f0
18 changed files with 803 additions and 38 deletions
+5 -5
View File
@@ -44,8 +44,8 @@ func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
_, store, history := busStoreForTest(t)
ctx := t.Context()
told := &Told{}
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
told, looks := &Told{}, NewMemoryLooks()
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told, Looks: looks})
if _, err := k.Observe(ctx, silent("ace")); err != nil {
t.Fatal(err)
}
@@ -54,7 +54,7 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
}
k.Close(context.Background())
again := NewKeeper(ctx, Options{Store: store, History: history})
again := NewKeeper(ctx, Options{Store: store, History: history, Looks: looks})
defer again.Close(context.Background())
open, err := again.Open(ctx)
if err != nil {
@@ -67,13 +67,13 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision+7); err != ErrMoved {
t.Fatalf("a write at a stale revision answered %v", err)
}
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
t.Fatalf("creating an open condition answered %v", err)
}
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
if err := store.Delete(ctx, "machine.ace.silent", e.Revision+7); err != ErrMoved {
t.Fatalf("a delete at a stale revision answered %v", err)
}
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
+6 -1
View File
@@ -138,7 +138,9 @@ type Condition struct {
// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
Needs string `json:"needs"`
Actions []Action `json:"actions"`
// Evidence is the newest observations, at most KeptEvidence, newest first.
// Evidence is the newest observations, at most KeptEvidence, newest first. It, LastObserved and
// Observations are the condition's looks (looks.go): kept in the controller's store, never in the
// conditions bucket, and put back on every condition the keeper reads and every event it says.
Evidence []Evidence `json:"evidence"`
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
Source string `json:"source"`
@@ -167,6 +169,9 @@ type Condition struct {
// Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written
// by a controller serving without the lease, or before the lease existed.
Epoch uint64 `json:"epoch"`
// LooksUnread says why Observations, LastObserved and Evidence are not known: they are kept in the
// controller's store, not the bucket (looks.go), and could not be read from it. Empty when they were.
LooksUnread string `json:"looks-unread,omitempty"`
}
// SilencedAt says whether a person's silence is in force at a moment.
+163
View File
@@ -0,0 +1,163 @@
package conditions
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
)
// Looks are what an open condition's observations say between its transitions: how many there were,
// when the newest was, and the newest evidence (to-be 45 §2). They change on every look, so **they are
// not kept in the conditions bucket**, which is written only when what a condition says changes
// (novox/hq ADR 0289 §1 and §4, issue 420): a watcher of the bucket heard thousands of writes a day for a
// dozen conditions that did not change. The controller's store keeps them (LookStore), and the keeper
// puts them back on every condition it reads and every event it says, so `conditions` and the events
// still carry them.
type Looks struct {
Observations int `json:"observations"`
LastObserved time.Time `json:"last-observed"`
Evidence []Evidence `json:"evidence"`
}
// LookStore is where the looks of the open conditions are kept, one row per key.
type LookStore interface {
// KeepLooks writes a condition's looks, replacing what it held.
KeepLooks(ctx context.Context, key string, l Looks) error
// Looked is one condition's looks; false when none are kept.
Looked(ctx context.Context, key string) (Looks, bool, error)
// AllLooks is every condition's looks. An error is an error, never "none kept" (ADR 0227 rule 4).
AllLooks(ctx context.Context) (map[string]Looks, error)
// ForgetLooks removes a condition's looks: it cleared.
ForgetLooks(ctx context.Context, key string) error
}
// lookFields are a condition's fields that are its looks, or say they could not be read: left out of
// the bucket's value.
var lookFields = []string{"observations", "last-observed", "evidence", "looks-unread"}
// looks is the condition's looks.
func (c Condition) looks() Looks {
return Looks{Observations: c.Observations, LastObserved: c.LastObserved, Evidence: c.Evidence}
}
// withLooks puts looks on the condition.
func (c *Condition) withLooks(l Looks) {
c.Observations, c.LastObserved, c.Evidence = l.Observations, l.LastObserved, l.Evidence
}
// BucketValue is a condition as the conditions bucket keeps it: everything but its looks. Its keys are in
// one order, so the same condition is the same bytes and an unchanged one is not written again.
func BucketValue(c Condition) ([]byte, error) {
whole, err := json.Marshal(c)
if err != nil {
return nil, err
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(whole, &fields); err != nil {
return nil, err
}
for _, f := range lookFields {
delete(fields, f)
}
return json.Marshal(fields)
}
// attachLooks puts each condition's kept looks on it. A condition the store keeps no looks for keeps
// what its value says (one written before the looks moved out of the bucket says them); one that says
// none is said to have none kept. Looks that cannot be read are said on each condition, never left
// as a count of nothing.
func attachLooks(ctx context.Context, looks LookStore, list []Condition) {
if looks == nil {
return
}
all, err := looks.AllLooks(ctx)
for i := range list {
if err != nil {
list[i].LooksUnread = "its observations could not be read from the controller's store: " + err.Error()
continue
}
l, found := all[list[i].Key]
attachOne(&list[i], l, found)
}
}
func attachOne(c *Condition, l Looks, found bool) {
switch {
case found:
c.withLooks(l)
case c.LastObserved.IsZero():
c.LooksUnread = "the controller's store keeps no observations for it"
}
}
// MemoryLooks keeps looks in this process: for tests, and for a keeper given no store of looks.
type MemoryLooks struct {
mu sync.Mutex
looks map[string]Looks
// Kept counts the writes, so a test can say a look was kept.
kept int
}
// NewMemoryLooks is an empty store of looks.
func NewMemoryLooks() *MemoryLooks { return &MemoryLooks{looks: map[string]Looks{}} }
func (m *MemoryLooks) KeepLooks(_ context.Context, key string, l Looks) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.looks == nil {
m.looks = map[string]Looks{}
}
l.Evidence = append([]Evidence(nil), l.Evidence...)
m.looks[key] = l
m.kept++
return nil
}
func (m *MemoryLooks) Looked(_ context.Context, key string) (Looks, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
l, ok := m.looks[key]
l.Evidence = append([]Evidence(nil), l.Evidence...)
return l, ok, nil
}
func (m *MemoryLooks) AllLooks(context.Context) (map[string]Looks, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := make(map[string]Looks, len(m.looks))
for k, l := range m.looks {
l.Evidence = append([]Evidence(nil), l.Evidence...)
out[k] = l
}
return out, nil
}
func (m *MemoryLooks) ForgetLooks(_ context.Context, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.looks, key)
return nil
}
// LooksKept is how many times looks were written.
func (m *MemoryLooks) LooksKept() int {
m.mu.Lock()
defer m.mu.Unlock()
return m.kept
}
// NoLooks is a store of looks that cannot be reached, saying why on every read and write: a command
// whose process could not open the controller's store still reads and writes the bucket, and says its
// observations unread rather than none.
type NoLooks struct{ Err error }
func (n NoLooks) err() error { return fmt.Errorf("the controller's store is not open here: %w", n.Err) }
func (n NoLooks) KeepLooks(context.Context, string, Looks) error { return n.err() }
func (n NoLooks) Looked(context.Context, string) (Looks, bool, error) {
return Looks{}, false, n.err()
}
func (n NoLooks) AllLooks(context.Context) (map[string]Looks, error) { return nil, n.err() }
func (n NoLooks) ForgetLooks(context.Context, string) error { return n.err() }
+161
View File
@@ -0,0 +1,161 @@
package conditions
import (
"context"
"encoding/json"
"errors"
"testing"
"time"
)
// **A look that sees a condition as it was writes nothing to the bucket** (novox/hq issue 420, ADR 0289
// §1 and §4): on 2026-10-10 the bucket took 131 647 writes in four days for twelve open conditions,
// because every look rewrote the count, the last look and the evidence. The bucket is written when the
// condition is raised and when what it says changes; the looks are kept beside it, and `conditions` and
// the events still carry them.
func TestALookThatChangesNothingWritesNothingToTheBucket(t *testing.T) {
k, store, told, c := keeper(t)
ctx := t.Context()
for range 5 {
if _, err := k.Observe(ctx, silent("ace")); err != nil {
t.Fatal(err)
}
c.pass(30 * time.Second)
}
if store.Writes() != 1 {
t.Fatalf("five looks at one unchanged condition wrote the bucket %d times, want 1 (its raising)", store.Writes())
}
got, open, err := k.Get(ctx, "machine.ace.silent")
if err != nil || !open {
t.Fatalf("open %v: %v", open, err)
}
if got.Observations != 5 || len(got.Evidence) != 5 || !got.LastObserved.Equal(c.now().Add(-30*time.Second)) ||
got.LooksUnread != "" {
t.Fatalf("the verb reads %d observation(s), %d evidence line(s), last %s (%q)", got.Observations,
len(got.Evidence), got.LastObserved, got.LooksUnread)
}
list, err := k.Open(ctx)
if err != nil || len(list) != 1 || list[0].Observations != 5 {
t.Fatalf("the list reads %+v: %v", list, err)
}
// The bucket's value holds what the condition says, and none of its looks.
e, _, _ := store.Get(ctx, "machine.ace.silent")
var fields map[string]json.RawMessage
if err := json.Unmarshal(e.Value, &fields); err != nil {
t.Fatal(err)
}
for _, f := range lookFields {
if _, held := fields[f]; held {
t.Errorf("the bucket's value holds %q, which changes on every look", f)
}
}
for _, f := range []string{"key", "kind", "severity", "resolver", "silenced", "raised", "headline", "needs", "epoch"} {
if _, held := fields[f]; !held {
t.Errorf("the bucket's value lacks %q", f)
}
}
// A change of what it says is written, once, and its event carries the looks.
worse := silent("ace")
worse.Severity = Urgent
if _, err := k.Observe(ctx, worse); err != nil {
t.Fatal(err)
}
if _, err := k.Observe(ctx, worse); err != nil {
t.Fatal(err)
}
if store.Writes() != 2 {
t.Fatalf("a change of severity and one more look wrote the bucket %d times in all, want 2", store.Writes())
}
said := settled(t, told, 2)
if said[1].Change != ChangeSeverity || said[1].Observations != 6 || len(said[1].Evidence) != 6 {
t.Fatalf("the severity's event says %s with %d observation(s) and %d evidence line(s)", said[1].Change,
said[1].Observations, len(said[1].Evidence))
}
// Its clearing forgets its looks, and its event carries them as they were.
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
t.Fatalf("cleared %v: %v", cleared, err)
}
if _, kept, _ := store.Looked(ctx, "machine.ace.silent"); kept {
t.Fatal("a cleared condition's looks are still kept")
}
said = settled(t, told, 3)
if said[2].Change != ChangeCleared || said[2].Observations != 7 {
t.Fatalf("the clearing says %s with %d observation(s)", said[2].Change, said[2].Observations)
}
}
// **A condition written before its looks moved out keeps its count**: its value says them, the first
// look counts on from there, and writes the value once without them.
func TestAConditionWrittenWithItsLooksMovesThemOutOnce(t *testing.T) {
k, store, _, c := keeper(t)
ctx := t.Context()
old := Condition{Key: "machine.ace.silent", Kind: "silent", Subject: Subject{Scope: ScopeMachine, ID: "ace", Machine: "ace"},
Severity: Warning, Summary: "ace has not been heard from", Source: "S1", Raised: c.now().Add(-time.Hour),
LastObserved: c.now().Add(-time.Minute), Observations: 7003, Count: 1, Resolver: ResolverSelf,
Evidence: []Evidence{{At: c.now().Add(-time.Minute), Said: "ace has not been heard from"}}}
body, _ := json.Marshal(old)
if err := store.Create(ctx, old.Key, body); err != nil {
t.Fatal(err)
}
if got, _, _ := k.Get(ctx, old.Key); got.Observations != 7003 || got.LooksUnread != "" {
t.Fatalf("before its first look it reads %d observation(s) (%q)", got.Observations, got.LooksUnread)
}
for range 3 {
if _, err := k.Observe(ctx, silent("ace")); err != nil {
t.Fatal(err)
}
c.pass(30 * time.Second)
}
if store.Writes() != 2 {
t.Fatalf("three looks wrote the bucket %d times after its creation, want 1", store.Writes()-1)
}
got, _, _ := k.Get(ctx, old.Key)
if got.Observations != 7006 || len(got.Evidence) != 4 {
t.Fatalf("it counts %d observation(s) and %d evidence line(s), want 7006 and 4", got.Observations, len(got.Evidence))
}
}
// failingLooks is a controller's store that is away.
type failingLooks struct{}
func (failingLooks) KeepLooks(context.Context, string, Looks) error { return errors.New("store away") }
func (failingLooks) Looked(context.Context, string) (Looks, bool, error) {
return Looks{}, false, errors.New("store away")
}
func (failingLooks) AllLooks(context.Context) (map[string]Looks, error) {
return nil, errors.New("store away")
}
func (failingLooks) ForgetLooks(context.Context, string) error { return errors.New("store away") }
// **A condition is raised while the controller's store is away**, and says its looks are unread rather
// than none: a store gone is exactly what a condition must still be raised for.
func TestLooksThatCannotBeKeptAreSaidNotCountedAsNone(t *testing.T) {
store, told := NewInMemory(), &Told{}
var said []string
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Looks: failingLooks{},
Say: func(f string, a ...any) { said = append(said, f) }})
t.Cleanup(func() { k.Close(context.Background()) })
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
t.Fatalf("a condition was not raised for a store away: %v", err)
}
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
t.Fatal(err)
}
got, open, err := k.Get(t.Context(), "machine.ace.silent")
if err != nil || !open || got.LooksUnread == "" {
t.Fatalf("open %v, looks unread %q: %v", open, got.LooksUnread, err)
}
list, err := k.Open(t.Context())
if err != nil || len(list) != 1 || list[0].LooksUnread == "" {
t.Fatalf("the list reads %+v: %v", list, err)
}
if e := settled(t, told, 1); e[0].LooksUnread == "" {
t.Fatal("the raising says nothing of its looks being unkept")
}
if len(said) == 0 {
t.Fatal("looks that could not be kept were not said")
}
}
+18 -2
View File
@@ -12,10 +12,14 @@ import (
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
type InMemory struct {
// MemoryLooks keeps its conditions' looks, as the controller's store does beside the bucket.
MemoryLooks
mu sync.Mutex
values map[string]Entry
revision uint64
events []Event
// writes counts every create, update and delete, so a test can say a look wrote nothing.
writes int
events []Event
// fail, when set, is what every read and write answers: a store that is away. It is set only
// through SetFail, under the lock, because the keeper's telling goroutine reads it while a test
// takes the store away.
@@ -23,7 +27,9 @@ type InMemory struct {
}
// NewInMemory is an empty store.
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
func NewInMemory() *InMemory {
return &InMemory{values: map[string]Entry{}, MemoryLooks: MemoryLooks{looks: map[string]Looks{}}}
}
// SetFail makes every read and write answer err from now on, or none when err is nil.
func (m *InMemory) SetFail(err error) {
@@ -52,10 +58,18 @@ func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
return ErrMoved
}
m.revision++
m.writes++
m.values[key] = Entry{Value: value, Revision: m.revision}
return nil
}
// Writes is how many creates, updates and deletes the store took.
func (m *InMemory) Writes() int {
m.mu.Lock()
defer m.mu.Unlock()
return m.writes
}
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
m.mu.Lock()
defer m.mu.Unlock()
@@ -66,6 +80,7 @@ func (m *InMemory) Update(_ context.Context, key string, value []byte, revision
return ErrMoved
}
m.revision++
m.writes++
m.values[key] = Entry{Value: value, Revision: m.revision}
return nil
}
@@ -79,6 +94,7 @@ func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error
if e, ok := m.values[key]; !ok || e.Revision != revision {
return ErrMoved
}
m.writes++
delete(m.values, key)
return nil
}
+94 -16
View File
@@ -1,6 +1,7 @@
package conditions
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -53,6 +54,7 @@ const Seat = "mesh-controller"
// Keeper raises, observes, silences and clears conditions, and says each transition.
type Keeper struct {
store Backend
looks LookStore
history History
teller Teller
now func() time.Time
@@ -91,7 +93,10 @@ type clearing struct {
// Options are what a Keeper is made with.
type Options struct {
Store Backend
Store Backend
// Looks is where the open conditions' looks are kept (looks.go): the controller's store. Nil takes the
// Store's own when it keeps them (the in-memory store does), and otherwise keeps them in this process.
Looks LookStore
History History
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
Teller Teller
@@ -111,7 +116,7 @@ var TellFor = 10 * time.Minute
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
// that cleared just before this controller started and is raised again now is a reopening.
func NewKeeper(ctx context.Context, o Options) *Keeper {
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
k := &Keeper{store: o.Store, looks: o.Looks, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
epoch: o.Epoch,
cleared: map[string]clearing{}, reworded: map[string]bool{}, out: make(chan Event, 1024), drained: make(chan struct{})}
if k.now == nil {
@@ -120,6 +125,13 @@ func NewKeeper(ctx context.Context, o Options) *Keeper {
if k.say == nil {
k.say = func(string, ...any) {}
}
if k.looks == nil {
if own, ok := o.Store.(LookStore); ok {
k.looks = own
} else {
k.looks = NewMemoryLooks()
}
}
if k.history != nil {
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
for _, e := range recent {
@@ -218,7 +230,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if err := k.stamp(&c); err != nil {
return Condition{}, err
}
body, err := json.Marshal(c)
body, err := BucketValue(c)
if err != nil {
return Condition{}, err
}
@@ -227,6 +239,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
} else if err != nil {
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
}
k.keepLooks(ctx, &c)
k.mu.Lock()
delete(k.cleared, key)
k.mu.Unlock()
@@ -237,6 +250,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if err := json.Unmarshal(entry.Value, &c); err != nil {
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
looked := k.attach(ctx, &c)
var changes []Event
// A healer's budget spent made it urgent; the watchdog seeing it again does not undo that.
if o.Severity != c.Severity && !(c.Escalated() && o.Severity != Urgent) {
@@ -278,14 +292,22 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if err := k.stamp(&c); err != nil {
return Condition{}, err
}
body, err := json.Marshal(c)
body, err := BucketValue(c)
if err != nil {
return Condition{}, err
}
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
continue
} else if err != nil {
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
// Written only when what the condition says changed (novox/hq ADR 0289 §1, issue 420): a look
// that saw it as it was changes only its looks, which the controller's store keeps.
if !bytes.Equal(body, entry.Value) {
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
continue
} else if err != nil {
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
}
}
// Looks read as unreadable are not written over: a count of one would replace the count kept.
if fresh && looked {
k.keepLooks(ctx, &c)
}
for _, e := range changes {
e.At, e.Condition = now, c
@@ -318,6 +340,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo
// Unreadable is not resolved: kept, and said, rather than removed unread.
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
}
k.attach(ctx, &c)
if err := k.stamp(&c); err != nil {
return false, err
}
@@ -329,6 +352,10 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo
} else if err != nil {
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
}
if err := k.looks.ForgetLooks(ctx, key); err != nil {
k.say("the observations of the condition %s, which cleared, could not be forgotten in the controller's "+
"store; a raising of it counts from one all the same: %v", key, err)
}
now := k.now().UTC()
k.mu.Lock()
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), gaps: c.Gaps, silenced: c.Silenced, tried: c.Tried}
@@ -402,12 +429,13 @@ func (k *Keeper) amend(ctx context.Context, key string, change func(*Condition,
if err := json.Unmarshal(entry.Value, &c); err != nil {
return Condition{}, false, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
k.attach(ctx, &c)
now := k.now().UTC()
changes := change(&c, now)
if err := k.stamp(&c); err != nil {
return Condition{}, false, err
}
body, err := json.Marshal(c)
body, err := BucketValue(c)
if err != nil {
return Condition{}, false, err
}
@@ -479,12 +507,13 @@ func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, w
if err := json.Unmarshal(entry.Value, &c); err != nil {
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
k.attach(ctx, &c)
now := k.now().UTC()
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
if err := k.stamp(&c); err != nil {
return Condition{}, err
}
body, err := json.Marshal(c)
body, err := BucketValue(c)
if err != nil {
return Condition{}, err
}
@@ -526,12 +555,13 @@ func (k *Keeper) EndSilences(ctx context.Context) error {
if held.Silenced == nil || now.Before(held.Silenced.Until) {
break
}
k.attach(ctx, &held)
was := held.Silenced.Why
held.Silenced = nil
if err := k.stamp(&held); err != nil {
return err
}
body, err := json.Marshal(held)
body, err := BucketValue(held)
if err != nil {
return err
}
@@ -549,12 +579,12 @@ func (k *Keeper) EndSilences(ctx context.Context) error {
// Open is every open condition, urgent first and then oldest first.
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
return Read(ctx, k.store)
return ReadWith(ctx, k.store, k.looks)
}
// Get is one open condition.
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
return ReadOne(ctx, k.store, key)
return ReadOneWith(ctx, k.store, k.looks, key)
}
// HistorySince is every transition from a moment, oldest first.
@@ -565,9 +595,16 @@ func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, er
return k.history.Since(ctx, since)
}
// Read is every open condition in a store, in the order status says them. A value that cannot be
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
// Read is every open condition in a store, in the order status says them, with its looks when the
// store keeps them. A value that cannot be read is an error naming its key, never a condition left out
// (ADR 0227 rule 4).
func Read(ctx context.Context, store Backend) ([]Condition, error) {
looks, _ := store.(LookStore)
return ReadWith(ctx, store, looks)
}
// ReadWith is Read with each condition's looks from where they are kept (looks.go); nil reads none.
func ReadWith(ctx context.Context, store Backend, looks LookStore) ([]Condition, error) {
all, err := store.All(ctx)
if err != nil {
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
@@ -580,12 +617,19 @@ func Read(ctx context.Context, store Backend) ([]Condition, error) {
}
out = append(out, c)
}
attachLooks(ctx, looks, out)
Order(out)
return out, nil
}
// ReadOne is one open condition from a store.
// ReadOne is one open condition from a store, with its looks when the store keeps them.
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
looks, _ := store.(LookStore)
return ReadOneWith(ctx, store, looks, key)
}
// ReadOneWith is ReadOne with its looks from where they are kept; nil reads none.
func ReadOneWith(ctx context.Context, store Backend, looks LookStore, key string) (Condition, bool, error) {
e, found, err := store.Get(ctx, key)
if err != nil || !found {
return Condition{}, found, err
@@ -594,9 +638,43 @@ func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, e
if err := json.Unmarshal(e.Value, &c); err != nil {
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
}
if looks != nil {
l, kept, err := looks.Looked(ctx, key)
if err != nil {
c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error()
} else {
attachOne(&c, l, kept)
}
}
return c, true, nil
}
// attach puts a condition's kept looks on it, as read from the bucket. False when they could not be
// read, which is said: the condition then carries why, and its looks are not written over.
func (k *Keeper) attach(ctx context.Context, c *Condition) bool {
l, kept, err := k.looks.Looked(ctx, c.Key)
if err != nil {
c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error()
k.say("the observations of the condition %s could not be read from the controller's store, so this look "+
"is not counted: %v", c.Key, err)
return false
}
attachOne(c, l, kept)
return true
}
// keepLooks writes a condition's looks, saying a write that failed: the condition is raised or kept in the
// bucket all the same, and this look is not counted.
func (k *Keeper) keepLooks(ctx context.Context, c *Condition) {
if err := k.looks.KeepLooks(ctx, c.Key, c.looks()); err == nil {
c.LooksUnread = ""
} else {
c.LooksUnread = "its observations could not be kept in the controller's store: " + err.Error()
k.say("the observations of the condition %s could not be kept in the controller's store, so this look "+
"is not counted: %v", c.Key, err)
}
}
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
func (k *Keeper) tell(e Event) {
+6 -2
View File
@@ -277,11 +277,15 @@ func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
}
}}
k.store = racing
got, err := k.Observe(ctx, silent("ace"))
// An observation that changes what the condition says, so it writes (one that changes nothing writes
// nothing, novox/hq issue 420).
worse := silent("ace")
worse.Severity = Urgent
got, err := k.Observe(ctx, worse)
if err != nil {
t.Fatal(err)
}
if got.Silenced == nil || got.Observations != 2 {
if got.Silenced == nil || got.Observations != 2 || got.Severity != Urgent {
t.Fatalf("the observation overwrote the silence: %+v", got)
}
}