diff --git a/cmd/mesh-controller/condition_looks.go b/cmd/mesh-controller/condition_looks.go new file mode 100644 index 00000000..74dc3902 --- /dev/null +++ b/cmd/mesh-controller/condition_looks.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The looks of the open conditions — how often each was observed, when last, its newest evidence — are +// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket +// is written when what a condition says changes, never for a look that saw it as it was. + +// storeLooks keeps the conditions' looks in the inventory. +type storeLooks struct{ inv *inventory.Inventory } + +func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error { + evidence, err := json.Marshal(l.Evidence) + if err != nil { + return err + } + return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations, + LastObserved: l.LastObserved, Evidence: evidence}) +} + +func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) { + row, kept, err := s.inv.ConditionLooksOf(ctx, key) + if err != nil || !kept { + return conditions.Looks{}, kept, err + } + l, err := looksOf(row) + return l, err == nil, err +} + +func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) { + rows, err := s.inv.AllConditionLooks(ctx) + if err != nil { + return nil, err + } + out := make(map[string]conditions.Looks, len(rows)) + for key, row := range rows { + l, err := looksOf(row) + if err != nil { + return nil, err + } + out[key] = l + } + return out, nil +} + +func (s storeLooks) ForgetLooks(ctx context.Context, key string) error { + return s.inv.ForgetConditionLooks(ctx, key) +} + +func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) { + l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()} + if len(row.Evidence) > 0 { + if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil { + return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err) + } + } + return l, nil +} + +// looksHere is where a process that is not the serving controller reads the conditions' looks: the store, +// opened for it, closed by the function returned. A store that cannot be opened is said on every condition +// read through it, never read as none. +func looksHere(ctx context.Context) (conditions.LookStore, func()) { + inv, err := openInventory(ctx) + if err != nil { + return conditions.NoLooks{Err: err}, func() {} + } + return storeLooks{inv}, inv.Close +} diff --git a/cmd/mesh-controller/condition_looks_test.go b/cmd/mesh-controller/condition_looks_test.go new file mode 100644 index 00000000..1c79015a --- /dev/null +++ b/cmd/mesh-controller/condition_looks_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when +// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a +// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003 +// times in two days. +func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) { + inv := inventory.ForTest(t) + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + if err := serving.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx := t.Context() + store, history, err := conditions.OnTheBus(ctx, serving.Conn()) + if err != nil { + t.Fatal(err) + } + k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}}) + defer k.Close(context.Background()) + look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor", + Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"} + for range 4 { + if _, err := k.Observe(ctx, look); err != nil { + t.Fatal(err) + } + } + + api, err := jetstream.New(serving.Conn()) + if err != nil { + t.Fatal(err) + } + stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket) + if err != nil { + t.Fatal(err) + } + info, err := stream.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.State.LastSeq != 1 { + t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq) + } + + // The verb as a process other than the serving controller: it reads the bucket and the store. + keptBefore := conditionsFrom + conditionsFrom = nil + defer func() { conditionsFrom = keptBefore }() + t.Setenv(broker.NATSVar, bus.ClientURL()) + t.Setenv(broker.CertificateVar, "") + var shown bytes.Buffer + if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil { + t.Fatal(err) + } + var c conditions.Condition + if err := json.Unmarshal(shown.Bytes(), &c); err != nil { + t.Fatalf("%v: %s", err, shown.String()) + } + if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" || + time.Since(c.LastObserved) > time.Minute { + t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations, + len(c.Evidence), c.LastObserved, c.LooksUnread) + } + listed, err := openConditions(ctx) + if err != nil || len(listed) != 1 || listed[0].Observations != 4 { + t.Fatalf("conditions lists %+v: %v", listed, err) + } +} diff --git a/cmd/mesh-controller/conditions.go b/cmd/mesh-controller/conditions.go index 0bc24496..8c64b3a9 100644 --- a/cmd/mesh-controller/conditions.go +++ b/cmd/mesh-controller/conditions.go @@ -24,16 +24,18 @@ import ( // **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every // core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised // by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold -// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and -// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an +// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its +// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go, +// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an // observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a // while, with a reason, and that is recorded as a hand act. // conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own. var conditionsFrom *conditions.Keeper -// keeperOn is a keeper over the store on a connection, saying its transitions on that connection. -func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) { +// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with +// the conditions' looks kept in looks (condition_looks.go). +func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) { store, history, err := conditions.OnTheBus(ctx, conn) if err != nil { return nil, err @@ -42,7 +44,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) if err != nil { return nil, err } - return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, + return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks, Teller: link.OverNATS{Conn: conn, JS: js}, Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }, // What status leads with changed: composed again soon (a nudge outside the serving controller @@ -58,8 +60,10 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error { if conditionsFrom != nil { return f(conditionsFrom) } + looks, closeLooks := looksHere(ctx) + defer closeLooks() return onTheBus(func(conn *nats.Conn) error { - k, err := keeperOn(ctx, conn) + k, err := keeperOn(ctx, conn, looks) if err != nil { return err } @@ -82,6 +86,8 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) { return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err) } var out []conditions.Condition + looks, closeLooks := looksHere(ctx) + defer closeLooks() err := onTheBus(func(conn *nats.Conn) error { store, _, err := conditions.OnTheBus(ctx, conn) if err != nil { @@ -89,7 +95,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) { } reading, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - out, err = conditions.Read(reading, store) + out, err = conditions.ReadWith(reading, store, looks) return err }) return out, err @@ -252,12 +258,14 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error { if conditionsFrom != nil { c, found, err = conditionsFrom.Get(ctx, key) } else { + looks, closeLooks := looksHere(ctx) + defer closeLooks() err = onTheBus(func(conn *nats.Conn) error { store, _, err := conditions.OnTheBus(ctx, conn) if err != nil { return err } - c, found, err = conditions.ReadOne(ctx, store, key) + c, found, err = conditions.ReadOneWith(ctx, store, looks, key) return err }) } diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index b7f48946..a79e5675 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -651,7 +651,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) { open.Close() return nil, nil, err } - k, err := keeperOn(ctx, js.Conn()) + k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory}) if err != nil { js.Close() open.Close() diff --git a/cmd/mesh-controller/replay327_test.go b/cmd/mesh-controller/replay327_test.go index 559debe9..73f2cc14 100644 --- a/cmd/mesh-controller/replay327_test.go +++ b/cmd/mesh-controller/replay327_test.go @@ -11,6 +11,7 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/testbus" ) @@ -30,7 +31,7 @@ func TestReplay327(t *testing.T) { if err := serving.EnsureControllerBuckets(); err != nil { t.Fatal(err) } - keeper, err := keeperOn(context.Background(), serving.Conn()) + keeper, err := keeperOn(context.Background(), serving.Conn(), conditions.NewMemoryLooks()) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 7ccb352d..58692163 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -704,7 +704,7 @@ func later(a, b time.Time) time.Time { // self-check, for the serving controller; the returned function stops them. Nil when the store could // not be opened, which is said. func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() { - keeper, err := keeperOn(ctx, bus.Conn) + keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory}) if err != nil { fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+ "status says the conditions cannot be read: %v\n", err) diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 32065552..c99dcde6 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -108,7 +108,8 @@ var WritersTable = []WriterRow{ {State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision", KeptIn: "the controller's store", Others: "read through plans"}, {State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " + - ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads", + ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " + + "controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads", Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController}, {State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket, Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController}, diff --git a/internal/conditions/bus_test.go b/internal/conditions/bus_test.go index f7f2a71c..7f945c83 100644 --- a/internal/conditions/bus_test.go +++ b/internal/conditions/bus_test.go @@ -44,8 +44,8 @@ func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) { func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { _, store, history := busStoreForTest(t) ctx := t.Context() - told := &Told{} - k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told}) + told, looks := &Told{}, NewMemoryLooks() + k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told, Looks: looks}) if _, err := k.Observe(ctx, silent("ace")); err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { } k.Close(context.Background()) - again := NewKeeper(ctx, Options{Store: store, History: history}) + again := NewKeeper(ctx, Options{Store: store, History: history, Looks: looks}) defer again.Close(context.Background()) open, err := again.Open(ctx) if err != nil { @@ -67,13 +67,13 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved { + if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision+7); err != ErrMoved { t.Fatalf("a write at a stale revision answered %v", err) } if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved { t.Fatalf("creating an open condition answered %v", err) } - if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved { + if err := store.Delete(ctx, "machine.ace.silent", e.Revision+7); err != ErrMoved { t.Fatalf("a delete at a stale revision answered %v", err) } if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared { diff --git a/internal/conditions/condition.go b/internal/conditions/condition.go index 8bf9a195..0b7bf5bb 100644 --- a/internal/conditions/condition.go +++ b/internal/conditions/condition.go @@ -138,7 +138,9 @@ type Condition struct { // opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go). Needs string `json:"needs"` Actions []Action `json:"actions"` - // Evidence is the newest observations, at most KeptEvidence, newest first. + // Evidence is the newest observations, at most KeptEvidence, newest first. It, LastObserved and + // Observations are the condition's looks (looks.go): kept in the controller's store, never in the + // conditions bucket, and put back on every condition the keeper reads and every event it says. Evidence []Evidence `json:"evidence"` // Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. Source string `json:"source"` @@ -167,6 +169,9 @@ type Condition struct { // Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written // by a controller serving without the lease, or before the lease existed. Epoch uint64 `json:"epoch"` + // LooksUnread says why Observations, LastObserved and Evidence are not known: they are kept in the + // controller's store, not the bucket (looks.go), and could not be read from it. Empty when they were. + LooksUnread string `json:"looks-unread,omitempty"` } // SilencedAt says whether a person's silence is in force at a moment. diff --git a/internal/conditions/looks.go b/internal/conditions/looks.go new file mode 100644 index 00000000..600073ff --- /dev/null +++ b/internal/conditions/looks.go @@ -0,0 +1,163 @@ +package conditions + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" +) + +// Looks are what an open condition's observations say between its transitions: how many there were, +// when the newest was, and the newest evidence (to-be 45 §2). They change on every look, so **they are +// not kept in the conditions bucket**, which is written only when what a condition says changes +// (novox/hq ADR 0289 §1 and §4, issue 420): a watcher of the bucket heard thousands of writes a day for a +// dozen conditions that did not change. The controller's store keeps them (LookStore), and the keeper +// puts them back on every condition it reads and every event it says, so `conditions` and the events +// still carry them. +type Looks struct { + Observations int `json:"observations"` + LastObserved time.Time `json:"last-observed"` + Evidence []Evidence `json:"evidence"` +} + +// LookStore is where the looks of the open conditions are kept, one row per key. +type LookStore interface { + // KeepLooks writes a condition's looks, replacing what it held. + KeepLooks(ctx context.Context, key string, l Looks) error + // Looked is one condition's looks; false when none are kept. + Looked(ctx context.Context, key string) (Looks, bool, error) + // AllLooks is every condition's looks. An error is an error, never "none kept" (ADR 0227 rule 4). + AllLooks(ctx context.Context) (map[string]Looks, error) + // ForgetLooks removes a condition's looks: it cleared. + ForgetLooks(ctx context.Context, key string) error +} + +// lookFields are a condition's fields that are its looks, or say they could not be read: left out of +// the bucket's value. +var lookFields = []string{"observations", "last-observed", "evidence", "looks-unread"} + +// looks is the condition's looks. +func (c Condition) looks() Looks { + return Looks{Observations: c.Observations, LastObserved: c.LastObserved, Evidence: c.Evidence} +} + +// withLooks puts looks on the condition. +func (c *Condition) withLooks(l Looks) { + c.Observations, c.LastObserved, c.Evidence = l.Observations, l.LastObserved, l.Evidence +} + +// BucketValue is a condition as the conditions bucket keeps it: everything but its looks. Its keys are in +// one order, so the same condition is the same bytes and an unchanged one is not written again. +func BucketValue(c Condition) ([]byte, error) { + whole, err := json.Marshal(c) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(whole, &fields); err != nil { + return nil, err + } + for _, f := range lookFields { + delete(fields, f) + } + return json.Marshal(fields) +} + +// attachLooks puts each condition's kept looks on it. A condition the store keeps no looks for keeps +// what its value says (one written before the looks moved out of the bucket says them); one that says +// none is said to have none kept. Looks that cannot be read are said on each condition, never left +// as a count of nothing. +func attachLooks(ctx context.Context, looks LookStore, list []Condition) { + if looks == nil { + return + } + all, err := looks.AllLooks(ctx) + for i := range list { + if err != nil { + list[i].LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + continue + } + l, found := all[list[i].Key] + attachOne(&list[i], l, found) + } +} + +func attachOne(c *Condition, l Looks, found bool) { + switch { + case found: + c.withLooks(l) + case c.LastObserved.IsZero(): + c.LooksUnread = "the controller's store keeps no observations for it" + } +} + +// MemoryLooks keeps looks in this process: for tests, and for a keeper given no store of looks. +type MemoryLooks struct { + mu sync.Mutex + looks map[string]Looks + // Kept counts the writes, so a test can say a look was kept. + kept int +} + +// NewMemoryLooks is an empty store of looks. +func NewMemoryLooks() *MemoryLooks { return &MemoryLooks{looks: map[string]Looks{}} } + +func (m *MemoryLooks) KeepLooks(_ context.Context, key string, l Looks) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.looks == nil { + m.looks = map[string]Looks{} + } + l.Evidence = append([]Evidence(nil), l.Evidence...) + m.looks[key] = l + m.kept++ + return nil +} + +func (m *MemoryLooks) Looked(_ context.Context, key string) (Looks, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + l, ok := m.looks[key] + l.Evidence = append([]Evidence(nil), l.Evidence...) + return l, ok, nil +} + +func (m *MemoryLooks) AllLooks(context.Context) (map[string]Looks, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make(map[string]Looks, len(m.looks)) + for k, l := range m.looks { + l.Evidence = append([]Evidence(nil), l.Evidence...) + out[k] = l + } + return out, nil +} + +func (m *MemoryLooks) ForgetLooks(_ context.Context, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.looks, key) + return nil +} + +// LooksKept is how many times looks were written. +func (m *MemoryLooks) LooksKept() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.kept +} + +// NoLooks is a store of looks that cannot be reached, saying why on every read and write: a command +// whose process could not open the controller's store still reads and writes the bucket, and says its +// observations unread rather than none. +type NoLooks struct{ Err error } + +func (n NoLooks) err() error { return fmt.Errorf("the controller's store is not open here: %w", n.Err) } + +func (n NoLooks) KeepLooks(context.Context, string, Looks) error { return n.err() } +func (n NoLooks) Looked(context.Context, string) (Looks, bool, error) { + return Looks{}, false, n.err() +} +func (n NoLooks) AllLooks(context.Context) (map[string]Looks, error) { return nil, n.err() } +func (n NoLooks) ForgetLooks(context.Context, string) error { return n.err() } diff --git a/internal/conditions/looks_test.go b/internal/conditions/looks_test.go new file mode 100644 index 00000000..d3f13ed5 --- /dev/null +++ b/internal/conditions/looks_test.go @@ -0,0 +1,161 @@ +package conditions + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" +) + +// **A look that sees a condition as it was writes nothing to the bucket** (novox/hq issue 420, ADR 0289 +// §1 and §4): on 2026-10-10 the bucket took 131 647 writes in four days for twelve open conditions, +// because every look rewrote the count, the last look and the evidence. The bucket is written when the +// condition is raised and when what it says changes; the looks are kept beside it, and `conditions` and +// the events still carry them. +func TestALookThatChangesNothingWritesNothingToTheBucket(t *testing.T) { + k, store, told, c := keeper(t) + ctx := t.Context() + for range 5 { + if _, err := k.Observe(ctx, silent("ace")); err != nil { + t.Fatal(err) + } + c.pass(30 * time.Second) + } + if store.Writes() != 1 { + t.Fatalf("five looks at one unchanged condition wrote the bucket %d times, want 1 (its raising)", store.Writes()) + } + got, open, err := k.Get(ctx, "machine.ace.silent") + if err != nil || !open { + t.Fatalf("open %v: %v", open, err) + } + if got.Observations != 5 || len(got.Evidence) != 5 || !got.LastObserved.Equal(c.now().Add(-30*time.Second)) || + got.LooksUnread != "" { + t.Fatalf("the verb reads %d observation(s), %d evidence line(s), last %s (%q)", got.Observations, + len(got.Evidence), got.LastObserved, got.LooksUnread) + } + list, err := k.Open(ctx) + if err != nil || len(list) != 1 || list[0].Observations != 5 { + t.Fatalf("the list reads %+v: %v", list, err) + } + + // The bucket's value holds what the condition says, and none of its looks. + e, _, _ := store.Get(ctx, "machine.ace.silent") + var fields map[string]json.RawMessage + if err := json.Unmarshal(e.Value, &fields); err != nil { + t.Fatal(err) + } + for _, f := range lookFields { + if _, held := fields[f]; held { + t.Errorf("the bucket's value holds %q, which changes on every look", f) + } + } + for _, f := range []string{"key", "kind", "severity", "resolver", "silenced", "raised", "headline", "needs", "epoch"} { + if _, held := fields[f]; !held { + t.Errorf("the bucket's value lacks %q", f) + } + } + + // A change of what it says is written, once, and its event carries the looks. + worse := silent("ace") + worse.Severity = Urgent + if _, err := k.Observe(ctx, worse); err != nil { + t.Fatal(err) + } + if _, err := k.Observe(ctx, worse); err != nil { + t.Fatal(err) + } + if store.Writes() != 2 { + t.Fatalf("a change of severity and one more look wrote the bucket %d times in all, want 2", store.Writes()) + } + said := settled(t, told, 2) + if said[1].Change != ChangeSeverity || said[1].Observations != 6 || len(said[1].Evidence) != 6 { + t.Fatalf("the severity's event says %s with %d observation(s) and %d evidence line(s)", said[1].Change, + said[1].Observations, len(said[1].Evidence)) + } + + // Its clearing forgets its looks, and its event carries them as they were. + if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared { + t.Fatalf("cleared %v: %v", cleared, err) + } + if _, kept, _ := store.Looked(ctx, "machine.ace.silent"); kept { + t.Fatal("a cleared condition's looks are still kept") + } + said = settled(t, told, 3) + if said[2].Change != ChangeCleared || said[2].Observations != 7 { + t.Fatalf("the clearing says %s with %d observation(s)", said[2].Change, said[2].Observations) + } +} + +// **A condition written before its looks moved out keeps its count**: its value says them, the first +// look counts on from there, and writes the value once without them. +func TestAConditionWrittenWithItsLooksMovesThemOutOnce(t *testing.T) { + k, store, _, c := keeper(t) + ctx := t.Context() + old := Condition{Key: "machine.ace.silent", Kind: "silent", Subject: Subject{Scope: ScopeMachine, ID: "ace", Machine: "ace"}, + Severity: Warning, Summary: "ace has not been heard from", Source: "S1", Raised: c.now().Add(-time.Hour), + LastObserved: c.now().Add(-time.Minute), Observations: 7003, Count: 1, Resolver: ResolverSelf, + Evidence: []Evidence{{At: c.now().Add(-time.Minute), Said: "ace has not been heard from"}}} + body, _ := json.Marshal(old) + if err := store.Create(ctx, old.Key, body); err != nil { + t.Fatal(err) + } + if got, _, _ := k.Get(ctx, old.Key); got.Observations != 7003 || got.LooksUnread != "" { + t.Fatalf("before its first look it reads %d observation(s) (%q)", got.Observations, got.LooksUnread) + } + for range 3 { + if _, err := k.Observe(ctx, silent("ace")); err != nil { + t.Fatal(err) + } + c.pass(30 * time.Second) + } + if store.Writes() != 2 { + t.Fatalf("three looks wrote the bucket %d times after its creation, want 1", store.Writes()-1) + } + got, _, _ := k.Get(ctx, old.Key) + if got.Observations != 7006 || len(got.Evidence) != 4 { + t.Fatalf("it counts %d observation(s) and %d evidence line(s), want 7006 and 4", got.Observations, len(got.Evidence)) + } +} + +// failingLooks is a controller's store that is away. +type failingLooks struct{} + +func (failingLooks) KeepLooks(context.Context, string, Looks) error { return errors.New("store away") } +func (failingLooks) Looked(context.Context, string) (Looks, bool, error) { + return Looks{}, false, errors.New("store away") +} +func (failingLooks) AllLooks(context.Context) (map[string]Looks, error) { + return nil, errors.New("store away") +} +func (failingLooks) ForgetLooks(context.Context, string) error { return errors.New("store away") } + +// **A condition is raised while the controller's store is away**, and says its looks are unread rather +// than none: a store gone is exactly what a condition must still be raised for. +func TestLooksThatCannotBeKeptAreSaidNotCountedAsNone(t *testing.T) { + store, told := NewInMemory(), &Told{} + var said []string + k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Looks: failingLooks{}, + Say: func(f string, a ...any) { said = append(said, f) }}) + t.Cleanup(func() { k.Close(context.Background()) }) + if _, err := k.Observe(t.Context(), silent("ace")); err != nil { + t.Fatalf("a condition was not raised for a store away: %v", err) + } + if _, err := k.Observe(t.Context(), silent("ace")); err != nil { + t.Fatal(err) + } + got, open, err := k.Get(t.Context(), "machine.ace.silent") + if err != nil || !open || got.LooksUnread == "" { + t.Fatalf("open %v, looks unread %q: %v", open, got.LooksUnread, err) + } + list, err := k.Open(t.Context()) + if err != nil || len(list) != 1 || list[0].LooksUnread == "" { + t.Fatalf("the list reads %+v: %v", list, err) + } + if e := settled(t, told, 1); e[0].LooksUnread == "" { + t.Fatal("the raising says nothing of its looks being unkept") + } + if len(said) == 0 { + t.Fatal("looks that could not be kept were not said") + } +} diff --git a/internal/conditions/memory.go b/internal/conditions/memory.go index 0c3da547..0974cf15 100644 --- a/internal/conditions/memory.go +++ b/internal/conditions/memory.go @@ -12,10 +12,14 @@ import ( // InMemory is a store and a history held in this process: for tests, and for nothing else — a // condition kept here is forgotten by a restart, which is the fault the store exists to remove. type InMemory struct { + // MemoryLooks keeps its conditions' looks, as the controller's store does beside the bucket. + MemoryLooks mu sync.Mutex values map[string]Entry revision uint64 - events []Event + // writes counts every create, update and delete, so a test can say a look wrote nothing. + writes int + events []Event // fail, when set, is what every read and write answers: a store that is away. It is set only // through SetFail, under the lock, because the keeper's telling goroutine reads it while a test // takes the store away. @@ -23,7 +27,9 @@ type InMemory struct { } // NewInMemory is an empty store. -func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} } +func NewInMemory() *InMemory { + return &InMemory{values: map[string]Entry{}, MemoryLooks: MemoryLooks{looks: map[string]Looks{}}} +} // SetFail makes every read and write answer err from now on, or none when err is nil. func (m *InMemory) SetFail(err error) { @@ -52,10 +58,18 @@ func (m *InMemory) Create(_ context.Context, key string, value []byte) error { return ErrMoved } m.revision++ + m.writes++ m.values[key] = Entry{Value: value, Revision: m.revision} return nil } +// Writes is how many creates, updates and deletes the store took. +func (m *InMemory) Writes() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.writes +} + func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error { m.mu.Lock() defer m.mu.Unlock() @@ -66,6 +80,7 @@ func (m *InMemory) Update(_ context.Context, key string, value []byte, revision return ErrMoved } m.revision++ + m.writes++ m.values[key] = Entry{Value: value, Revision: m.revision} return nil } @@ -79,6 +94,7 @@ func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error if e, ok := m.values[key]; !ok || e.Revision != revision { return ErrMoved } + m.writes++ delete(m.values, key) return nil } diff --git a/internal/conditions/store.go b/internal/conditions/store.go index c36c1a4f..0311d60d 100644 --- a/internal/conditions/store.go +++ b/internal/conditions/store.go @@ -1,6 +1,7 @@ package conditions import ( + "bytes" "context" "encoding/json" "errors" @@ -53,6 +54,7 @@ const Seat = "mesh-controller" // Keeper raises, observes, silences and clears conditions, and says each transition. type Keeper struct { store Backend + looks LookStore history History teller Teller now func() time.Time @@ -91,7 +93,10 @@ type clearing struct { // Options are what a Keeper is made with. type Options struct { - Store Backend + Store Backend + // Looks is where the open conditions' looks are kept (looks.go): the controller's store. Nil takes the + // Store's own when it keeps them (the in-memory store does), and otherwise keeps them in this process. + Looks LookStore History History // Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on). Teller Teller @@ -111,7 +116,7 @@ var TellFor = 10 * time.Minute // NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition // that cleared just before this controller started and is raised again now is a reopening. func NewKeeper(ctx context.Context, o Options) *Keeper { - k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed, + k := &Keeper{store: o.Store, looks: o.Looks, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed, epoch: o.Epoch, cleared: map[string]clearing{}, reworded: map[string]bool{}, out: make(chan Event, 1024), drained: make(chan struct{})} if k.now == nil { @@ -120,6 +125,13 @@ func NewKeeper(ctx context.Context, o Options) *Keeper { if k.say == nil { k.say = func(string, ...any) {} } + if k.looks == nil { + if own, ok := o.Store.(LookStore); ok { + k.looks = own + } else { + k.looks = NewMemoryLooks() + } + } if k.history != nil { if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil { for _, e := range recent { @@ -218,7 +230,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } @@ -227,6 +239,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) } else if err != nil { return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err) } + k.keepLooks(ctx, &c) k.mu.Lock() delete(k.cleared, key) k.mu.Unlock() @@ -237,6 +250,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + looked := k.attach(ctx, &c) var changes []Event // A healer's budget spent made it urgent; the watchdog seeing it again does not undo that. if o.Severity != c.Severity && !(c.Escalated() && o.Severity != Urgent) { @@ -278,14 +292,22 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } - if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) { - continue - } else if err != nil { - return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err) + // Written only when what the condition says changed (novox/hq ADR 0289 §1, issue 420): a look + // that saw it as it was changes only its looks, which the controller's store keeps. + if !bytes.Equal(body, entry.Value) { + if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) { + continue + } else if err != nil { + return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err) + } + } + // Looks read as unreadable are not written over: a count of one would replace the count kept. + if fresh && looked { + k.keepLooks(ctx, &c) } for _, e := range changes { e.At, e.Condition = now, c @@ -318,6 +340,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo // Unreadable is not resolved: kept, and said, rather than removed unread. return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err) } + k.attach(ctx, &c) if err := k.stamp(&c); err != nil { return false, err } @@ -329,6 +352,10 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo } else if err != nil { return false, fmt.Errorf("clearing the condition %s: %w", key, err) } + if err := k.looks.ForgetLooks(ctx, key); err != nil { + k.say("the observations of the condition %s, which cleared, could not be forgotten in the controller's "+ + "store; a raising of it counts from one all the same: %v", key, err) + } now := k.now().UTC() k.mu.Lock() k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), gaps: c.Gaps, silenced: c.Silenced, tried: c.Tried} @@ -402,12 +429,13 @@ func (k *Keeper) amend(ctx context.Context, key string, change func(*Condition, if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, false, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + k.attach(ctx, &c) now := k.now().UTC() changes := change(&c, now) if err := k.stamp(&c); err != nil { return Condition{}, false, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, false, err } @@ -479,12 +507,13 @@ func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, w if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + k.attach(ctx, &c) now := k.now().UTC() c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now} if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } @@ -526,12 +555,13 @@ func (k *Keeper) EndSilences(ctx context.Context) error { if held.Silenced == nil || now.Before(held.Silenced.Until) { break } + k.attach(ctx, &held) was := held.Silenced.Why held.Silenced = nil if err := k.stamp(&held); err != nil { return err } - body, err := json.Marshal(held) + body, err := BucketValue(held) if err != nil { return err } @@ -549,12 +579,12 @@ func (k *Keeper) EndSilences(ctx context.Context) error { // Open is every open condition, urgent first and then oldest first. func (k *Keeper) Open(ctx context.Context) ([]Condition, error) { - return Read(ctx, k.store) + return ReadWith(ctx, k.store, k.looks) } // Get is one open condition. func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) { - return ReadOne(ctx, k.store, key) + return ReadOneWith(ctx, k.store, k.looks, key) } // HistorySince is every transition from a moment, oldest first. @@ -565,9 +595,16 @@ func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, er return k.history.Since(ctx, since) } -// Read is every open condition in a store, in the order status says them. A value that cannot be -// read is an error naming its key, never a condition left out (ADR 0227 rule 4). +// Read is every open condition in a store, in the order status says them, with its looks when the +// store keeps them. A value that cannot be read is an error naming its key, never a condition left out +// (ADR 0227 rule 4). func Read(ctx context.Context, store Backend) ([]Condition, error) { + looks, _ := store.(LookStore) + return ReadWith(ctx, store, looks) +} + +// ReadWith is Read with each condition's looks from where they are kept (looks.go); nil reads none. +func ReadWith(ctx context.Context, store Backend, looks LookStore) ([]Condition, error) { all, err := store.All(ctx) if err != nil { return nil, fmt.Errorf("the open conditions cannot be read: %w", err) @@ -580,12 +617,19 @@ func Read(ctx context.Context, store Backend) ([]Condition, error) { } out = append(out, c) } + attachLooks(ctx, looks, out) Order(out) return out, nil } -// ReadOne is one open condition from a store. +// ReadOne is one open condition from a store, with its looks when the store keeps them. func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) { + looks, _ := store.(LookStore) + return ReadOneWith(ctx, store, looks, key) +} + +// ReadOneWith is ReadOne with its looks from where they are kept; nil reads none. +func ReadOneWith(ctx context.Context, store Backend, looks LookStore, key string) (Condition, bool, error) { e, found, err := store.Get(ctx, key) if err != nil || !found { return Condition{}, found, err @@ -594,9 +638,43 @@ func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, e if err := json.Unmarshal(e.Value, &c); err != nil { return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err) } + if looks != nil { + l, kept, err := looks.Looked(ctx, key) + if err != nil { + c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + } else { + attachOne(&c, l, kept) + } + } return c, true, nil } +// attach puts a condition's kept looks on it, as read from the bucket. False when they could not be +// read, which is said: the condition then carries why, and its looks are not written over. +func (k *Keeper) attach(ctx context.Context, c *Condition) bool { + l, kept, err := k.looks.Looked(ctx, c.Key) + if err != nil { + c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + k.say("the observations of the condition %s could not be read from the controller's store, so this look "+ + "is not counted: %v", c.Key, err) + return false + } + attachOne(c, l, kept) + return true +} + +// keepLooks writes a condition's looks, saying a write that failed: the condition is raised or kept in the +// bucket all the same, and this look is not counted. +func (k *Keeper) keepLooks(ctx context.Context, c *Condition) { + if err := k.looks.KeepLooks(ctx, c.Key, c.looks()); err == nil { + c.LooksUnread = "" + } else { + c.LooksUnread = "its observations could not be kept in the controller's store: " + err.Error() + k.say("the observations of the condition %s could not be kept in the controller's store, so this look "+ + "is not counted: %v", c.Key, err) + } +} + // tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is // full is a bus away for a long time, and the transition is said lost rather than holding a watchdog. func (k *Keeper) tell(e Event) { diff --git a/internal/conditions/store_test.go b/internal/conditions/store_test.go index d83716ef..f1d49a28 100644 --- a/internal/conditions/store_test.go +++ b/internal/conditions/store_test.go @@ -277,11 +277,15 @@ func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) { } }} k.store = racing - got, err := k.Observe(ctx, silent("ace")) + // An observation that changes what the condition says, so it writes (one that changes nothing writes + // nothing, novox/hq issue 420). + worse := silent("ace") + worse.Severity = Urgent + got, err := k.Observe(ctx, worse) if err != nil { t.Fatal(err) } - if got.Silenced == nil || got.Observations != 2 { + if got.Silenced == nil || got.Observations != 2 || got.Severity != Urgent { t.Fatalf("the observation overwrote the silence: %+v", got) } } diff --git a/internal/inventory/looks.go b/internal/inventory/looks.go new file mode 100644 index 00000000..3e310a0d --- /dev/null +++ b/internal/inventory/looks.go @@ -0,0 +1,90 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// The looks of the open conditions (novox/hq issue 420): see migration 0095. How many times each was +// observed, when last, and the newest evidence change on every look, so they are kept here rather than in +// the conditions bucket, which is written only when what a condition says changes (ADR 0289 §1). + +// ConditionLooks is one open condition's looks. Evidence is kept as the condition says it, newest first. +type ConditionLooks struct { + Key string + Observations int + LastObserved time.Time + Evidence json.RawMessage +} + +// KeepConditionLooks writes a condition's looks, replacing what its row held. Written under the lease: a +// controller that may not act keeps no looks. +func (i *Inventory) KeepConditionLooks(ctx context.Context, l ConditionLooks) error { + if l.Key == "" { + return errors.New("a condition's looks name its key") + } + epoch, err := i.actingEpoch(ctx) + if err != nil { + return err + } + evidence := l.Evidence + if len(evidence) == 0 { + evidence = json.RawMessage("[]") + } + _, err = i.store.Pool().Exec(ctx, + `insert into condition_looks (key, observations, last_observed, evidence, epoch, kept_at) + values ($1, $2, $3, $4, $5, now()) + on conflict (key) do update set observations = excluded.observations, + last_observed = excluded.last_observed, evidence = excluded.evidence, epoch = excluded.epoch, + kept_at = excluded.kept_at`, + l.Key, l.Observations, l.LastObserved, []byte(evidence), epoch) + return err +} + +// ConditionLooksOf is one condition's looks; false when none are kept. +func (i *Inventory) ConditionLooksOf(ctx context.Context, key string) (ConditionLooks, bool, error) { + l := ConditionLooks{Key: key} + var evidence []byte + err := i.store.Pool().QueryRow(ctx, + `select observations, last_observed, evidence from condition_looks where key = $1`, key). + Scan(&l.Observations, &l.LastObserved, &evidence) + if errors.Is(err, pgx.ErrNoRows) { + return ConditionLooks{}, false, nil + } + if err != nil { + return ConditionLooks{}, false, err + } + l.Evidence = evidence + return l, true, nil +} + +// AllConditionLooks is every kept condition's looks, by key. +func (i *Inventory) AllConditionLooks(ctx context.Context) (map[string]ConditionLooks, error) { + rows, err := i.store.Pool().Query(ctx, `select key, observations, last_observed, evidence from condition_looks`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]ConditionLooks{} + for rows.Next() { + var l ConditionLooks + var evidence []byte + if err := rows.Scan(&l.Key, &l.Observations, &l.LastObserved, &evidence); err != nil { + return nil, err + } + l.Evidence = evidence + out[l.Key] = l + } + return out, rows.Err() +} + +// ForgetConditionLooks removes a condition's looks: it cleared. Whatever the lease says by now, as a +// clearing is: what is not open has nothing to count. +func (i *Inventory) ForgetConditionLooks(ctx context.Context, key string) error { + _, err := i.store.Pool().Exec(ctx, `delete from condition_looks where key = $1`, key) + return err +} diff --git a/internal/inventory/looks_test.go b/internal/inventory/looks_test.go new file mode 100644 index 00000000..9b41fdda --- /dev/null +++ b/internal/inventory/looks_test.go @@ -0,0 +1,53 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" +) + +// **A condition's looks are kept in the store, replaced on every look and forgotten when it clears** +// (novox/hq issue 420): written under the lease, read back whole. +func TestAConditionsLooksAreKeptBesideTheBucket(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }) + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.silent", Observations: 1, + LastObserved: time.Now()}); err == nil { + t.Fatal("looks were kept by a process that may not act") + } + inv.ActsUnder(func(context.Context) (uint64, error) { return 57, nil }) + at := time.Now().UTC().Truncate(time.Microsecond) + for n := 1; n <= 3; n++ { + evidence, _ := json.Marshal([]map[string]any{{"at": at, "said": "ace has not been heard from"}}) + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.silent", Observations: n, + LastObserved: at, Evidence: evidence}); err != nil { + t.Fatal(err) + } + } + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.late", Observations: 2, + LastObserved: at}); err != nil { + t.Fatal(err) + } + got, kept, err := inv.ConditionLooksOf(ctx, "machine.ace.silent") + if err != nil || !kept || got.Observations != 3 || !got.LastObserved.Equal(at) { + t.Fatalf("read %+v kept %v: %v", got, kept, err) + } + var evidence []map[string]any + if err := json.Unmarshal(got.Evidence, &evidence); err != nil || len(evidence) != 1 { + t.Fatalf("evidence %s: %v", got.Evidence, err) + } + all, err := inv.AllConditionLooks(ctx) + if err != nil || len(all) != 2 || all["machine.ace.late"].Observations != 2 || + string(all["machine.ace.late"].Evidence) != "[]" { + t.Fatalf("all %+v: %v", all, err) + } + if err := inv.ForgetConditionLooks(ctx, "machine.ace.silent"); err != nil { + t.Fatal(err) + } + if _, kept, err := inv.ConditionLooksOf(ctx, "machine.ace.silent"); kept || err != nil { + t.Fatalf("a cleared condition's looks kept %v: %v", kept, err) + } +} diff --git a/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql b/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql new file mode 100644 index 00000000..ce81ccdd --- /dev/null +++ b/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql @@ -0,0 +1,20 @@ +-- The looks of an open condition are kept here, not in the conditions bucket (novox/hq issue 420, ADR 0289 +-- §1 and §4). +-- +-- On 2026-10-10 the bus had stored 131 647 writes to the conditions bucket in four days while twelve +-- conditions were open: every look rewrote a condition's value, because the value carried how many times it +-- had been observed, when last and the newest evidence. A watcher heard a write for every look, not for +-- every change. The bucket now holds what a condition says, written when that changes; how often and when +-- it was last seen, and what each look said, change on every look and are kept in this row, written by the +-- serving controller under its lease (`epoch`) and read back by `conditions` and the events it says. +-- +-- One row per open condition, removed when it clears. `evidence` is the newest observations, newest first, +-- as the condition's `evidence` says them: [{"at", "said"}]. +create table condition_looks ( + key text primary key, + observations integer not null check (observations >= 0), + last_observed timestamptz not null, + evidence jsonb not null default '[]', + epoch bigint, + kept_at timestamptz not null default now() +); diff --git a/testdata/store-tests b/testdata/store-tests index d604f6d9..98b101b2 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -12,3 +12,5 @@ ./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore ./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain ./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance +./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket +./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds