From dcdafd37f008690935a1560c79145025e4c3cd88 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 12:11:13 +0200 Subject: [PATCH] Keep a condition's looks in the store, so a look that changes nothing writes no bucket (issue 420) Every look rewrote the condition's value in the conditions bucket, because the value carried the count of looks, the last look and the newest evidence: 131 647 writes in four days for twelve open conditions, against ADR 0289's rule that a bucket is written only when what it describes changed. The looks now live in the controller's store (migration 0095, condition_looks), the bucket holds what a condition says and is written only when that changes, and the conditions verb and the condition events put the looks back. --- cmd/mesh-controller/condition_looks.go | 76 ++++++++ cmd/mesh-controller/condition_looks_test.go | 87 ++++++++++ cmd/mesh-controller/conditions.go | 24 ++- cmd/mesh-controller/doctor.go | 2 +- cmd/mesh-controller/replay327_test.go | 3 +- cmd/mesh-controller/watchdogs.go | 2 +- internal/broker/writers.go | 3 +- internal/conditions/bus_test.go | 10 +- internal/conditions/condition.go | 7 +- internal/conditions/looks.go | 163 ++++++++++++++++++ internal/conditions/looks_test.go | 161 +++++++++++++++++ internal/conditions/memory.go | 20 ++- internal/conditions/store.go | 110 ++++++++++-- internal/conditions/store_test.go | 8 +- internal/inventory/looks.go | 90 ++++++++++ internal/inventory/looks_test.go | 53 ++++++ ...tion-keeps-its-looks-beside-the-bucket.sql | 20 +++ testdata/store-tests | 2 + 18 files changed, 803 insertions(+), 38 deletions(-) create mode 100644 cmd/mesh-controller/condition_looks.go create mode 100644 cmd/mesh-controller/condition_looks_test.go create mode 100644 internal/conditions/looks.go create mode 100644 internal/conditions/looks_test.go create mode 100644 internal/inventory/looks.go create mode 100644 internal/inventory/looks_test.go create mode 100644 internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql diff --git a/cmd/mesh-controller/condition_looks.go b/cmd/mesh-controller/condition_looks.go new file mode 100644 index 00000000..74dc3902 --- /dev/null +++ b/cmd/mesh-controller/condition_looks.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The looks of the open conditions — how often each was observed, when last, its newest evidence — are +// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket +// is written when what a condition says changes, never for a look that saw it as it was. + +// storeLooks keeps the conditions' looks in the inventory. +type storeLooks struct{ inv *inventory.Inventory } + +func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error { + evidence, err := json.Marshal(l.Evidence) + if err != nil { + return err + } + return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations, + LastObserved: l.LastObserved, Evidence: evidence}) +} + +func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) { + row, kept, err := s.inv.ConditionLooksOf(ctx, key) + if err != nil || !kept { + return conditions.Looks{}, kept, err + } + l, err := looksOf(row) + return l, err == nil, err +} + +func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) { + rows, err := s.inv.AllConditionLooks(ctx) + if err != nil { + return nil, err + } + out := make(map[string]conditions.Looks, len(rows)) + for key, row := range rows { + l, err := looksOf(row) + if err != nil { + return nil, err + } + out[key] = l + } + return out, nil +} + +func (s storeLooks) ForgetLooks(ctx context.Context, key string) error { + return s.inv.ForgetConditionLooks(ctx, key) +} + +func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) { + l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()} + if len(row.Evidence) > 0 { + if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil { + return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err) + } + } + return l, nil +} + +// looksHere is where a process that is not the serving controller reads the conditions' looks: the store, +// opened for it, closed by the function returned. A store that cannot be opened is said on every condition +// read through it, never read as none. +func looksHere(ctx context.Context) (conditions.LookStore, func()) { + inv, err := openInventory(ctx) + if err != nil { + return conditions.NoLooks{Err: err}, func() {} + } + return storeLooks{inv}, inv.Close +} diff --git a/cmd/mesh-controller/condition_looks_test.go b/cmd/mesh-controller/condition_looks_test.go new file mode 100644 index 00000000..1c79015a --- /dev/null +++ b/cmd/mesh-controller/condition_looks_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when +// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a +// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003 +// times in two days. +func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) { + inv := inventory.ForTest(t) + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + if err := serving.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx := t.Context() + store, history, err := conditions.OnTheBus(ctx, serving.Conn()) + if err != nil { + t.Fatal(err) + } + k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}}) + defer k.Close(context.Background()) + look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor", + Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"} + for range 4 { + if _, err := k.Observe(ctx, look); err != nil { + t.Fatal(err) + } + } + + api, err := jetstream.New(serving.Conn()) + if err != nil { + t.Fatal(err) + } + stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket) + if err != nil { + t.Fatal(err) + } + info, err := stream.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.State.LastSeq != 1 { + t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq) + } + + // The verb as a process other than the serving controller: it reads the bucket and the store. + keptBefore := conditionsFrom + conditionsFrom = nil + defer func() { conditionsFrom = keptBefore }() + t.Setenv(broker.NATSVar, bus.ClientURL()) + t.Setenv(broker.CertificateVar, "") + var shown bytes.Buffer + if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil { + t.Fatal(err) + } + var c conditions.Condition + if err := json.Unmarshal(shown.Bytes(), &c); err != nil { + t.Fatalf("%v: %s", err, shown.String()) + } + if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" || + time.Since(c.LastObserved) > time.Minute { + t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations, + len(c.Evidence), c.LastObserved, c.LooksUnread) + } + listed, err := openConditions(ctx) + if err != nil || len(listed) != 1 || listed[0].Observations != 4 { + t.Fatalf("conditions lists %+v: %v", listed, err) + } +} diff --git a/cmd/mesh-controller/conditions.go b/cmd/mesh-controller/conditions.go index 0bc24496..8c64b3a9 100644 --- a/cmd/mesh-controller/conditions.go +++ b/cmd/mesh-controller/conditions.go @@ -24,16 +24,18 @@ import ( // **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every // core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised // by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold -// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and -// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an +// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its +// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go, +// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an // observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a // while, with a reason, and that is recorded as a hand act. // conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own. var conditionsFrom *conditions.Keeper -// keeperOn is a keeper over the store on a connection, saying its transitions on that connection. -func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) { +// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with +// the conditions' looks kept in looks (condition_looks.go). +func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) { store, history, err := conditions.OnTheBus(ctx, conn) if err != nil { return nil, err @@ -42,7 +44,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) if err != nil { return nil, err } - return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, + return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks, Teller: link.OverNATS{Conn: conn, JS: js}, Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }, // What status leads with changed: composed again soon (a nudge outside the serving controller @@ -58,8 +60,10 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error { if conditionsFrom != nil { return f(conditionsFrom) } + looks, closeLooks := looksHere(ctx) + defer closeLooks() return onTheBus(func(conn *nats.Conn) error { - k, err := keeperOn(ctx, conn) + k, err := keeperOn(ctx, conn, looks) if err != nil { return err } @@ -82,6 +86,8 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) { return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err) } var out []conditions.Condition + looks, closeLooks := looksHere(ctx) + defer closeLooks() err := onTheBus(func(conn *nats.Conn) error { store, _, err := conditions.OnTheBus(ctx, conn) if err != nil { @@ -89,7 +95,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) { } reading, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - out, err = conditions.Read(reading, store) + out, err = conditions.ReadWith(reading, store, looks) return err }) return out, err @@ -252,12 +258,14 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error { if conditionsFrom != nil { c, found, err = conditionsFrom.Get(ctx, key) } else { + looks, closeLooks := looksHere(ctx) + defer closeLooks() err = onTheBus(func(conn *nats.Conn) error { store, _, err := conditions.OnTheBus(ctx, conn) if err != nil { return err } - c, found, err = conditions.ReadOne(ctx, store, key) + c, found, err = conditions.ReadOneWith(ctx, store, looks, key) return err }) } diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index b7f48946..a79e5675 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -651,7 +651,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) { open.Close() return nil, nil, err } - k, err := keeperOn(ctx, js.Conn()) + k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory}) if err != nil { js.Close() open.Close() diff --git a/cmd/mesh-controller/replay327_test.go b/cmd/mesh-controller/replay327_test.go index 559debe9..73f2cc14 100644 --- a/cmd/mesh-controller/replay327_test.go +++ b/cmd/mesh-controller/replay327_test.go @@ -11,6 +11,7 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/testbus" ) @@ -30,7 +31,7 @@ func TestReplay327(t *testing.T) { if err := serving.EnsureControllerBuckets(); err != nil { t.Fatal(err) } - keeper, err := keeperOn(context.Background(), serving.Conn()) + keeper, err := keeperOn(context.Background(), serving.Conn(), conditions.NewMemoryLooks()) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 7ccb352d..58692163 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -704,7 +704,7 @@ func later(a, b time.Time) time.Time { // self-check, for the serving controller; the returned function stops them. Nil when the store could // not be opened, which is said. func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() { - keeper, err := keeperOn(ctx, bus.Conn) + keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory}) if err != nil { fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+ "status says the conditions cannot be read: %v\n", err) diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 32065552..c99dcde6 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -108,7 +108,8 @@ var WritersTable = []WriterRow{ {State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision", KeptIn: "the controller's store", Others: "read through plans"}, {State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " + - ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads", + ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " + + "controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads", Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController}, {State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket, Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController}, diff --git a/internal/conditions/bus_test.go b/internal/conditions/bus_test.go index f7f2a71c..7f945c83 100644 --- a/internal/conditions/bus_test.go +++ b/internal/conditions/bus_test.go @@ -44,8 +44,8 @@ func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) { func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { _, store, history := busStoreForTest(t) ctx := t.Context() - told := &Told{} - k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told}) + told, looks := &Told{}, NewMemoryLooks() + k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told, Looks: looks}) if _, err := k.Observe(ctx, silent("ace")); err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { } k.Close(context.Background()) - again := NewKeeper(ctx, Options{Store: store, History: history}) + again := NewKeeper(ctx, Options{Store: store, History: history, Looks: looks}) defer again.Close(context.Background()) open, err := again.Open(ctx) if err != nil { @@ -67,13 +67,13 @@ func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved { + if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision+7); err != ErrMoved { t.Fatalf("a write at a stale revision answered %v", err) } if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved { t.Fatalf("creating an open condition answered %v", err) } - if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved { + if err := store.Delete(ctx, "machine.ace.silent", e.Revision+7); err != ErrMoved { t.Fatalf("a delete at a stale revision answered %v", err) } if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared { diff --git a/internal/conditions/condition.go b/internal/conditions/condition.go index 8bf9a195..0b7bf5bb 100644 --- a/internal/conditions/condition.go +++ b/internal/conditions/condition.go @@ -138,7 +138,9 @@ type Condition struct { // opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go). Needs string `json:"needs"` Actions []Action `json:"actions"` - // Evidence is the newest observations, at most KeptEvidence, newest first. + // Evidence is the newest observations, at most KeptEvidence, newest first. It, LastObserved and + // Observations are the condition's looks (looks.go): kept in the controller's store, never in the + // conditions bucket, and put back on every condition the keeper reads and every event it says. Evidence []Evidence `json:"evidence"` // Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. Source string `json:"source"` @@ -167,6 +169,9 @@ type Condition struct { // Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written // by a controller serving without the lease, or before the lease existed. Epoch uint64 `json:"epoch"` + // LooksUnread says why Observations, LastObserved and Evidence are not known: they are kept in the + // controller's store, not the bucket (looks.go), and could not be read from it. Empty when they were. + LooksUnread string `json:"looks-unread,omitempty"` } // SilencedAt says whether a person's silence is in force at a moment. diff --git a/internal/conditions/looks.go b/internal/conditions/looks.go new file mode 100644 index 00000000..600073ff --- /dev/null +++ b/internal/conditions/looks.go @@ -0,0 +1,163 @@ +package conditions + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" +) + +// Looks are what an open condition's observations say between its transitions: how many there were, +// when the newest was, and the newest evidence (to-be 45 §2). They change on every look, so **they are +// not kept in the conditions bucket**, which is written only when what a condition says changes +// (novox/hq ADR 0289 §1 and §4, issue 420): a watcher of the bucket heard thousands of writes a day for a +// dozen conditions that did not change. The controller's store keeps them (LookStore), and the keeper +// puts them back on every condition it reads and every event it says, so `conditions` and the events +// still carry them. +type Looks struct { + Observations int `json:"observations"` + LastObserved time.Time `json:"last-observed"` + Evidence []Evidence `json:"evidence"` +} + +// LookStore is where the looks of the open conditions are kept, one row per key. +type LookStore interface { + // KeepLooks writes a condition's looks, replacing what it held. + KeepLooks(ctx context.Context, key string, l Looks) error + // Looked is one condition's looks; false when none are kept. + Looked(ctx context.Context, key string) (Looks, bool, error) + // AllLooks is every condition's looks. An error is an error, never "none kept" (ADR 0227 rule 4). + AllLooks(ctx context.Context) (map[string]Looks, error) + // ForgetLooks removes a condition's looks: it cleared. + ForgetLooks(ctx context.Context, key string) error +} + +// lookFields are a condition's fields that are its looks, or say they could not be read: left out of +// the bucket's value. +var lookFields = []string{"observations", "last-observed", "evidence", "looks-unread"} + +// looks is the condition's looks. +func (c Condition) looks() Looks { + return Looks{Observations: c.Observations, LastObserved: c.LastObserved, Evidence: c.Evidence} +} + +// withLooks puts looks on the condition. +func (c *Condition) withLooks(l Looks) { + c.Observations, c.LastObserved, c.Evidence = l.Observations, l.LastObserved, l.Evidence +} + +// BucketValue is a condition as the conditions bucket keeps it: everything but its looks. Its keys are in +// one order, so the same condition is the same bytes and an unchanged one is not written again. +func BucketValue(c Condition) ([]byte, error) { + whole, err := json.Marshal(c) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(whole, &fields); err != nil { + return nil, err + } + for _, f := range lookFields { + delete(fields, f) + } + return json.Marshal(fields) +} + +// attachLooks puts each condition's kept looks on it. A condition the store keeps no looks for keeps +// what its value says (one written before the looks moved out of the bucket says them); one that says +// none is said to have none kept. Looks that cannot be read are said on each condition, never left +// as a count of nothing. +func attachLooks(ctx context.Context, looks LookStore, list []Condition) { + if looks == nil { + return + } + all, err := looks.AllLooks(ctx) + for i := range list { + if err != nil { + list[i].LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + continue + } + l, found := all[list[i].Key] + attachOne(&list[i], l, found) + } +} + +func attachOne(c *Condition, l Looks, found bool) { + switch { + case found: + c.withLooks(l) + case c.LastObserved.IsZero(): + c.LooksUnread = "the controller's store keeps no observations for it" + } +} + +// MemoryLooks keeps looks in this process: for tests, and for a keeper given no store of looks. +type MemoryLooks struct { + mu sync.Mutex + looks map[string]Looks + // Kept counts the writes, so a test can say a look was kept. + kept int +} + +// NewMemoryLooks is an empty store of looks. +func NewMemoryLooks() *MemoryLooks { return &MemoryLooks{looks: map[string]Looks{}} } + +func (m *MemoryLooks) KeepLooks(_ context.Context, key string, l Looks) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.looks == nil { + m.looks = map[string]Looks{} + } + l.Evidence = append([]Evidence(nil), l.Evidence...) + m.looks[key] = l + m.kept++ + return nil +} + +func (m *MemoryLooks) Looked(_ context.Context, key string) (Looks, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + l, ok := m.looks[key] + l.Evidence = append([]Evidence(nil), l.Evidence...) + return l, ok, nil +} + +func (m *MemoryLooks) AllLooks(context.Context) (map[string]Looks, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make(map[string]Looks, len(m.looks)) + for k, l := range m.looks { + l.Evidence = append([]Evidence(nil), l.Evidence...) + out[k] = l + } + return out, nil +} + +func (m *MemoryLooks) ForgetLooks(_ context.Context, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.looks, key) + return nil +} + +// LooksKept is how many times looks were written. +func (m *MemoryLooks) LooksKept() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.kept +} + +// NoLooks is a store of looks that cannot be reached, saying why on every read and write: a command +// whose process could not open the controller's store still reads and writes the bucket, and says its +// observations unread rather than none. +type NoLooks struct{ Err error } + +func (n NoLooks) err() error { return fmt.Errorf("the controller's store is not open here: %w", n.Err) } + +func (n NoLooks) KeepLooks(context.Context, string, Looks) error { return n.err() } +func (n NoLooks) Looked(context.Context, string) (Looks, bool, error) { + return Looks{}, false, n.err() +} +func (n NoLooks) AllLooks(context.Context) (map[string]Looks, error) { return nil, n.err() } +func (n NoLooks) ForgetLooks(context.Context, string) error { return n.err() } diff --git a/internal/conditions/looks_test.go b/internal/conditions/looks_test.go new file mode 100644 index 00000000..d3f13ed5 --- /dev/null +++ b/internal/conditions/looks_test.go @@ -0,0 +1,161 @@ +package conditions + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" +) + +// **A look that sees a condition as it was writes nothing to the bucket** (novox/hq issue 420, ADR 0289 +// §1 and §4): on 2026-10-10 the bucket took 131 647 writes in four days for twelve open conditions, +// because every look rewrote the count, the last look and the evidence. The bucket is written when the +// condition is raised and when what it says changes; the looks are kept beside it, and `conditions` and +// the events still carry them. +func TestALookThatChangesNothingWritesNothingToTheBucket(t *testing.T) { + k, store, told, c := keeper(t) + ctx := t.Context() + for range 5 { + if _, err := k.Observe(ctx, silent("ace")); err != nil { + t.Fatal(err) + } + c.pass(30 * time.Second) + } + if store.Writes() != 1 { + t.Fatalf("five looks at one unchanged condition wrote the bucket %d times, want 1 (its raising)", store.Writes()) + } + got, open, err := k.Get(ctx, "machine.ace.silent") + if err != nil || !open { + t.Fatalf("open %v: %v", open, err) + } + if got.Observations != 5 || len(got.Evidence) != 5 || !got.LastObserved.Equal(c.now().Add(-30*time.Second)) || + got.LooksUnread != "" { + t.Fatalf("the verb reads %d observation(s), %d evidence line(s), last %s (%q)", got.Observations, + len(got.Evidence), got.LastObserved, got.LooksUnread) + } + list, err := k.Open(ctx) + if err != nil || len(list) != 1 || list[0].Observations != 5 { + t.Fatalf("the list reads %+v: %v", list, err) + } + + // The bucket's value holds what the condition says, and none of its looks. + e, _, _ := store.Get(ctx, "machine.ace.silent") + var fields map[string]json.RawMessage + if err := json.Unmarshal(e.Value, &fields); err != nil { + t.Fatal(err) + } + for _, f := range lookFields { + if _, held := fields[f]; held { + t.Errorf("the bucket's value holds %q, which changes on every look", f) + } + } + for _, f := range []string{"key", "kind", "severity", "resolver", "silenced", "raised", "headline", "needs", "epoch"} { + if _, held := fields[f]; !held { + t.Errorf("the bucket's value lacks %q", f) + } + } + + // A change of what it says is written, once, and its event carries the looks. + worse := silent("ace") + worse.Severity = Urgent + if _, err := k.Observe(ctx, worse); err != nil { + t.Fatal(err) + } + if _, err := k.Observe(ctx, worse); err != nil { + t.Fatal(err) + } + if store.Writes() != 2 { + t.Fatalf("a change of severity and one more look wrote the bucket %d times in all, want 2", store.Writes()) + } + said := settled(t, told, 2) + if said[1].Change != ChangeSeverity || said[1].Observations != 6 || len(said[1].Evidence) != 6 { + t.Fatalf("the severity's event says %s with %d observation(s) and %d evidence line(s)", said[1].Change, + said[1].Observations, len(said[1].Evidence)) + } + + // Its clearing forgets its looks, and its event carries them as they were. + if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared { + t.Fatalf("cleared %v: %v", cleared, err) + } + if _, kept, _ := store.Looked(ctx, "machine.ace.silent"); kept { + t.Fatal("a cleared condition's looks are still kept") + } + said = settled(t, told, 3) + if said[2].Change != ChangeCleared || said[2].Observations != 7 { + t.Fatalf("the clearing says %s with %d observation(s)", said[2].Change, said[2].Observations) + } +} + +// **A condition written before its looks moved out keeps its count**: its value says them, the first +// look counts on from there, and writes the value once without them. +func TestAConditionWrittenWithItsLooksMovesThemOutOnce(t *testing.T) { + k, store, _, c := keeper(t) + ctx := t.Context() + old := Condition{Key: "machine.ace.silent", Kind: "silent", Subject: Subject{Scope: ScopeMachine, ID: "ace", Machine: "ace"}, + Severity: Warning, Summary: "ace has not been heard from", Source: "S1", Raised: c.now().Add(-time.Hour), + LastObserved: c.now().Add(-time.Minute), Observations: 7003, Count: 1, Resolver: ResolverSelf, + Evidence: []Evidence{{At: c.now().Add(-time.Minute), Said: "ace has not been heard from"}}} + body, _ := json.Marshal(old) + if err := store.Create(ctx, old.Key, body); err != nil { + t.Fatal(err) + } + if got, _, _ := k.Get(ctx, old.Key); got.Observations != 7003 || got.LooksUnread != "" { + t.Fatalf("before its first look it reads %d observation(s) (%q)", got.Observations, got.LooksUnread) + } + for range 3 { + if _, err := k.Observe(ctx, silent("ace")); err != nil { + t.Fatal(err) + } + c.pass(30 * time.Second) + } + if store.Writes() != 2 { + t.Fatalf("three looks wrote the bucket %d times after its creation, want 1", store.Writes()-1) + } + got, _, _ := k.Get(ctx, old.Key) + if got.Observations != 7006 || len(got.Evidence) != 4 { + t.Fatalf("it counts %d observation(s) and %d evidence line(s), want 7006 and 4", got.Observations, len(got.Evidence)) + } +} + +// failingLooks is a controller's store that is away. +type failingLooks struct{} + +func (failingLooks) KeepLooks(context.Context, string, Looks) error { return errors.New("store away") } +func (failingLooks) Looked(context.Context, string) (Looks, bool, error) { + return Looks{}, false, errors.New("store away") +} +func (failingLooks) AllLooks(context.Context) (map[string]Looks, error) { + return nil, errors.New("store away") +} +func (failingLooks) ForgetLooks(context.Context, string) error { return errors.New("store away") } + +// **A condition is raised while the controller's store is away**, and says its looks are unread rather +// than none: a store gone is exactly what a condition must still be raised for. +func TestLooksThatCannotBeKeptAreSaidNotCountedAsNone(t *testing.T) { + store, told := NewInMemory(), &Told{} + var said []string + k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Looks: failingLooks{}, + Say: func(f string, a ...any) { said = append(said, f) }}) + t.Cleanup(func() { k.Close(context.Background()) }) + if _, err := k.Observe(t.Context(), silent("ace")); err != nil { + t.Fatalf("a condition was not raised for a store away: %v", err) + } + if _, err := k.Observe(t.Context(), silent("ace")); err != nil { + t.Fatal(err) + } + got, open, err := k.Get(t.Context(), "machine.ace.silent") + if err != nil || !open || got.LooksUnread == "" { + t.Fatalf("open %v, looks unread %q: %v", open, got.LooksUnread, err) + } + list, err := k.Open(t.Context()) + if err != nil || len(list) != 1 || list[0].LooksUnread == "" { + t.Fatalf("the list reads %+v: %v", list, err) + } + if e := settled(t, told, 1); e[0].LooksUnread == "" { + t.Fatal("the raising says nothing of its looks being unkept") + } + if len(said) == 0 { + t.Fatal("looks that could not be kept were not said") + } +} diff --git a/internal/conditions/memory.go b/internal/conditions/memory.go index 0c3da547..0974cf15 100644 --- a/internal/conditions/memory.go +++ b/internal/conditions/memory.go @@ -12,10 +12,14 @@ import ( // InMemory is a store and a history held in this process: for tests, and for nothing else — a // condition kept here is forgotten by a restart, which is the fault the store exists to remove. type InMemory struct { + // MemoryLooks keeps its conditions' looks, as the controller's store does beside the bucket. + MemoryLooks mu sync.Mutex values map[string]Entry revision uint64 - events []Event + // writes counts every create, update and delete, so a test can say a look wrote nothing. + writes int + events []Event // fail, when set, is what every read and write answers: a store that is away. It is set only // through SetFail, under the lock, because the keeper's telling goroutine reads it while a test // takes the store away. @@ -23,7 +27,9 @@ type InMemory struct { } // NewInMemory is an empty store. -func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} } +func NewInMemory() *InMemory { + return &InMemory{values: map[string]Entry{}, MemoryLooks: MemoryLooks{looks: map[string]Looks{}}} +} // SetFail makes every read and write answer err from now on, or none when err is nil. func (m *InMemory) SetFail(err error) { @@ -52,10 +58,18 @@ func (m *InMemory) Create(_ context.Context, key string, value []byte) error { return ErrMoved } m.revision++ + m.writes++ m.values[key] = Entry{Value: value, Revision: m.revision} return nil } +// Writes is how many creates, updates and deletes the store took. +func (m *InMemory) Writes() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.writes +} + func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error { m.mu.Lock() defer m.mu.Unlock() @@ -66,6 +80,7 @@ func (m *InMemory) Update(_ context.Context, key string, value []byte, revision return ErrMoved } m.revision++ + m.writes++ m.values[key] = Entry{Value: value, Revision: m.revision} return nil } @@ -79,6 +94,7 @@ func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error if e, ok := m.values[key]; !ok || e.Revision != revision { return ErrMoved } + m.writes++ delete(m.values, key) return nil } diff --git a/internal/conditions/store.go b/internal/conditions/store.go index c36c1a4f..0311d60d 100644 --- a/internal/conditions/store.go +++ b/internal/conditions/store.go @@ -1,6 +1,7 @@ package conditions import ( + "bytes" "context" "encoding/json" "errors" @@ -53,6 +54,7 @@ const Seat = "mesh-controller" // Keeper raises, observes, silences and clears conditions, and says each transition. type Keeper struct { store Backend + looks LookStore history History teller Teller now func() time.Time @@ -91,7 +93,10 @@ type clearing struct { // Options are what a Keeper is made with. type Options struct { - Store Backend + Store Backend + // Looks is where the open conditions' looks are kept (looks.go): the controller's store. Nil takes the + // Store's own when it keeps them (the in-memory store does), and otherwise keeps them in this process. + Looks LookStore History History // Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on). Teller Teller @@ -111,7 +116,7 @@ var TellFor = 10 * time.Minute // NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition // that cleared just before this controller started and is raised again now is a reopening. func NewKeeper(ctx context.Context, o Options) *Keeper { - k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed, + k := &Keeper{store: o.Store, looks: o.Looks, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed, epoch: o.Epoch, cleared: map[string]clearing{}, reworded: map[string]bool{}, out: make(chan Event, 1024), drained: make(chan struct{})} if k.now == nil { @@ -120,6 +125,13 @@ func NewKeeper(ctx context.Context, o Options) *Keeper { if k.say == nil { k.say = func(string, ...any) {} } + if k.looks == nil { + if own, ok := o.Store.(LookStore); ok { + k.looks = own + } else { + k.looks = NewMemoryLooks() + } + } if k.history != nil { if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil { for _, e := range recent { @@ -218,7 +230,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } @@ -227,6 +239,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) } else if err != nil { return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err) } + k.keepLooks(ctx, &c) k.mu.Lock() delete(k.cleared, key) k.mu.Unlock() @@ -237,6 +250,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + looked := k.attach(ctx, &c) var changes []Event // A healer's budget spent made it urgent; the watchdog seeing it again does not undo that. if o.Severity != c.Severity && !(c.Escalated() && o.Severity != Urgent) { @@ -278,14 +292,22 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } - if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) { - continue - } else if err != nil { - return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err) + // Written only when what the condition says changed (novox/hq ADR 0289 §1, issue 420): a look + // that saw it as it was changes only its looks, which the controller's store keeps. + if !bytes.Equal(body, entry.Value) { + if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) { + continue + } else if err != nil { + return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err) + } + } + // Looks read as unreadable are not written over: a count of one would replace the count kept. + if fresh && looked { + k.keepLooks(ctx, &c) } for _, e := range changes { e.At, e.Condition = now, c @@ -318,6 +340,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo // Unreadable is not resolved: kept, and said, rather than removed unread. return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err) } + k.attach(ctx, &c) if err := k.stamp(&c); err != nil { return false, err } @@ -329,6 +352,10 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo } else if err != nil { return false, fmt.Errorf("clearing the condition %s: %w", key, err) } + if err := k.looks.ForgetLooks(ctx, key); err != nil { + k.say("the observations of the condition %s, which cleared, could not be forgotten in the controller's "+ + "store; a raising of it counts from one all the same: %v", key, err) + } now := k.now().UTC() k.mu.Lock() k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), gaps: c.Gaps, silenced: c.Silenced, tried: c.Tried} @@ -402,12 +429,13 @@ func (k *Keeper) amend(ctx context.Context, key string, change func(*Condition, if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, false, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + k.attach(ctx, &c) now := k.now().UTC() changes := change(&c, now) if err := k.stamp(&c); err != nil { return Condition{}, false, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, false, err } @@ -479,12 +507,13 @@ func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, w if err := json.Unmarshal(entry.Value, &c); err != nil { return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) } + k.attach(ctx, &c) now := k.now().UTC() c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now} if err := k.stamp(&c); err != nil { return Condition{}, err } - body, err := json.Marshal(c) + body, err := BucketValue(c) if err != nil { return Condition{}, err } @@ -526,12 +555,13 @@ func (k *Keeper) EndSilences(ctx context.Context) error { if held.Silenced == nil || now.Before(held.Silenced.Until) { break } + k.attach(ctx, &held) was := held.Silenced.Why held.Silenced = nil if err := k.stamp(&held); err != nil { return err } - body, err := json.Marshal(held) + body, err := BucketValue(held) if err != nil { return err } @@ -549,12 +579,12 @@ func (k *Keeper) EndSilences(ctx context.Context) error { // Open is every open condition, urgent first and then oldest first. func (k *Keeper) Open(ctx context.Context) ([]Condition, error) { - return Read(ctx, k.store) + return ReadWith(ctx, k.store, k.looks) } // Get is one open condition. func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) { - return ReadOne(ctx, k.store, key) + return ReadOneWith(ctx, k.store, k.looks, key) } // HistorySince is every transition from a moment, oldest first. @@ -565,9 +595,16 @@ func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, er return k.history.Since(ctx, since) } -// Read is every open condition in a store, in the order status says them. A value that cannot be -// read is an error naming its key, never a condition left out (ADR 0227 rule 4). +// Read is every open condition in a store, in the order status says them, with its looks when the +// store keeps them. A value that cannot be read is an error naming its key, never a condition left out +// (ADR 0227 rule 4). func Read(ctx context.Context, store Backend) ([]Condition, error) { + looks, _ := store.(LookStore) + return ReadWith(ctx, store, looks) +} + +// ReadWith is Read with each condition's looks from where they are kept (looks.go); nil reads none. +func ReadWith(ctx context.Context, store Backend, looks LookStore) ([]Condition, error) { all, err := store.All(ctx) if err != nil { return nil, fmt.Errorf("the open conditions cannot be read: %w", err) @@ -580,12 +617,19 @@ func Read(ctx context.Context, store Backend) ([]Condition, error) { } out = append(out, c) } + attachLooks(ctx, looks, out) Order(out) return out, nil } -// ReadOne is one open condition from a store. +// ReadOne is one open condition from a store, with its looks when the store keeps them. func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) { + looks, _ := store.(LookStore) + return ReadOneWith(ctx, store, looks, key) +} + +// ReadOneWith is ReadOne with its looks from where they are kept; nil reads none. +func ReadOneWith(ctx context.Context, store Backend, looks LookStore, key string) (Condition, bool, error) { e, found, err := store.Get(ctx, key) if err != nil || !found { return Condition{}, found, err @@ -594,9 +638,43 @@ func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, e if err := json.Unmarshal(e.Value, &c); err != nil { return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err) } + if looks != nil { + l, kept, err := looks.Looked(ctx, key) + if err != nil { + c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + } else { + attachOne(&c, l, kept) + } + } return c, true, nil } +// attach puts a condition's kept looks on it, as read from the bucket. False when they could not be +// read, which is said: the condition then carries why, and its looks are not written over. +func (k *Keeper) attach(ctx context.Context, c *Condition) bool { + l, kept, err := k.looks.Looked(ctx, c.Key) + if err != nil { + c.LooksUnread = "its observations could not be read from the controller's store: " + err.Error() + k.say("the observations of the condition %s could not be read from the controller's store, so this look "+ + "is not counted: %v", c.Key, err) + return false + } + attachOne(c, l, kept) + return true +} + +// keepLooks writes a condition's looks, saying a write that failed: the condition is raised or kept in the +// bucket all the same, and this look is not counted. +func (k *Keeper) keepLooks(ctx context.Context, c *Condition) { + if err := k.looks.KeepLooks(ctx, c.Key, c.looks()); err == nil { + c.LooksUnread = "" + } else { + c.LooksUnread = "its observations could not be kept in the controller's store: " + err.Error() + k.say("the observations of the condition %s could not be kept in the controller's store, so this look "+ + "is not counted: %v", c.Key, err) + } +} + // tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is // full is a bus away for a long time, and the transition is said lost rather than holding a watchdog. func (k *Keeper) tell(e Event) { diff --git a/internal/conditions/store_test.go b/internal/conditions/store_test.go index d83716ef..f1d49a28 100644 --- a/internal/conditions/store_test.go +++ b/internal/conditions/store_test.go @@ -277,11 +277,15 @@ func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) { } }} k.store = racing - got, err := k.Observe(ctx, silent("ace")) + // An observation that changes what the condition says, so it writes (one that changes nothing writes + // nothing, novox/hq issue 420). + worse := silent("ace") + worse.Severity = Urgent + got, err := k.Observe(ctx, worse) if err != nil { t.Fatal(err) } - if got.Silenced == nil || got.Observations != 2 { + if got.Silenced == nil || got.Observations != 2 || got.Severity != Urgent { t.Fatalf("the observation overwrote the silence: %+v", got) } } diff --git a/internal/inventory/looks.go b/internal/inventory/looks.go new file mode 100644 index 00000000..3e310a0d --- /dev/null +++ b/internal/inventory/looks.go @@ -0,0 +1,90 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// The looks of the open conditions (novox/hq issue 420): see migration 0095. How many times each was +// observed, when last, and the newest evidence change on every look, so they are kept here rather than in +// the conditions bucket, which is written only when what a condition says changes (ADR 0289 §1). + +// ConditionLooks is one open condition's looks. Evidence is kept as the condition says it, newest first. +type ConditionLooks struct { + Key string + Observations int + LastObserved time.Time + Evidence json.RawMessage +} + +// KeepConditionLooks writes a condition's looks, replacing what its row held. Written under the lease: a +// controller that may not act keeps no looks. +func (i *Inventory) KeepConditionLooks(ctx context.Context, l ConditionLooks) error { + if l.Key == "" { + return errors.New("a condition's looks name its key") + } + epoch, err := i.actingEpoch(ctx) + if err != nil { + return err + } + evidence := l.Evidence + if len(evidence) == 0 { + evidence = json.RawMessage("[]") + } + _, err = i.store.Pool().Exec(ctx, + `insert into condition_looks (key, observations, last_observed, evidence, epoch, kept_at) + values ($1, $2, $3, $4, $5, now()) + on conflict (key) do update set observations = excluded.observations, + last_observed = excluded.last_observed, evidence = excluded.evidence, epoch = excluded.epoch, + kept_at = excluded.kept_at`, + l.Key, l.Observations, l.LastObserved, []byte(evidence), epoch) + return err +} + +// ConditionLooksOf is one condition's looks; false when none are kept. +func (i *Inventory) ConditionLooksOf(ctx context.Context, key string) (ConditionLooks, bool, error) { + l := ConditionLooks{Key: key} + var evidence []byte + err := i.store.Pool().QueryRow(ctx, + `select observations, last_observed, evidence from condition_looks where key = $1`, key). + Scan(&l.Observations, &l.LastObserved, &evidence) + if errors.Is(err, pgx.ErrNoRows) { + return ConditionLooks{}, false, nil + } + if err != nil { + return ConditionLooks{}, false, err + } + l.Evidence = evidence + return l, true, nil +} + +// AllConditionLooks is every kept condition's looks, by key. +func (i *Inventory) AllConditionLooks(ctx context.Context) (map[string]ConditionLooks, error) { + rows, err := i.store.Pool().Query(ctx, `select key, observations, last_observed, evidence from condition_looks`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]ConditionLooks{} + for rows.Next() { + var l ConditionLooks + var evidence []byte + if err := rows.Scan(&l.Key, &l.Observations, &l.LastObserved, &evidence); err != nil { + return nil, err + } + l.Evidence = evidence + out[l.Key] = l + } + return out, rows.Err() +} + +// ForgetConditionLooks removes a condition's looks: it cleared. Whatever the lease says by now, as a +// clearing is: what is not open has nothing to count. +func (i *Inventory) ForgetConditionLooks(ctx context.Context, key string) error { + _, err := i.store.Pool().Exec(ctx, `delete from condition_looks where key = $1`, key) + return err +} diff --git a/internal/inventory/looks_test.go b/internal/inventory/looks_test.go new file mode 100644 index 00000000..9b41fdda --- /dev/null +++ b/internal/inventory/looks_test.go @@ -0,0 +1,53 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" +) + +// **A condition's looks are kept in the store, replaced on every look and forgotten when it clears** +// (novox/hq issue 420): written under the lease, read back whole. +func TestAConditionsLooksAreKeptBesideTheBucket(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }) + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.silent", Observations: 1, + LastObserved: time.Now()}); err == nil { + t.Fatal("looks were kept by a process that may not act") + } + inv.ActsUnder(func(context.Context) (uint64, error) { return 57, nil }) + at := time.Now().UTC().Truncate(time.Microsecond) + for n := 1; n <= 3; n++ { + evidence, _ := json.Marshal([]map[string]any{{"at": at, "said": "ace has not been heard from"}}) + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.silent", Observations: n, + LastObserved: at, Evidence: evidence}); err != nil { + t.Fatal(err) + } + } + if err := inv.KeepConditionLooks(ctx, ConditionLooks{Key: "machine.ace.late", Observations: 2, + LastObserved: at}); err != nil { + t.Fatal(err) + } + got, kept, err := inv.ConditionLooksOf(ctx, "machine.ace.silent") + if err != nil || !kept || got.Observations != 3 || !got.LastObserved.Equal(at) { + t.Fatalf("read %+v kept %v: %v", got, kept, err) + } + var evidence []map[string]any + if err := json.Unmarshal(got.Evidence, &evidence); err != nil || len(evidence) != 1 { + t.Fatalf("evidence %s: %v", got.Evidence, err) + } + all, err := inv.AllConditionLooks(ctx) + if err != nil || len(all) != 2 || all["machine.ace.late"].Observations != 2 || + string(all["machine.ace.late"].Evidence) != "[]" { + t.Fatalf("all %+v: %v", all, err) + } + if err := inv.ForgetConditionLooks(ctx, "machine.ace.silent"); err != nil { + t.Fatal(err) + } + if _, kept, err := inv.ConditionLooksOf(ctx, "machine.ace.silent"); kept || err != nil { + t.Fatalf("a cleared condition's looks kept %v: %v", kept, err) + } +} diff --git a/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql b/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql new file mode 100644 index 00000000..ce81ccdd --- /dev/null +++ b/internal/inventory/migrations/0095-a-condition-keeps-its-looks-beside-the-bucket.sql @@ -0,0 +1,20 @@ +-- The looks of an open condition are kept here, not in the conditions bucket (novox/hq issue 420, ADR 0289 +-- §1 and §4). +-- +-- On 2026-10-10 the bus had stored 131 647 writes to the conditions bucket in four days while twelve +-- conditions were open: every look rewrote a condition's value, because the value carried how many times it +-- had been observed, when last and the newest evidence. A watcher heard a write for every look, not for +-- every change. The bucket now holds what a condition says, written when that changes; how often and when +-- it was last seen, and what each look said, change on every look and are kept in this row, written by the +-- serving controller under its lease (`epoch`) and read back by `conditions` and the events it says. +-- +-- One row per open condition, removed when it clears. `evidence` is the newest observations, newest first, +-- as the condition's `evidence` says them: [{"at", "said"}]. +create table condition_looks ( + key text primary key, + observations integer not null check (observations >= 0), + last_observed timestamptz not null, + evidence jsonb not null default '[]', + epoch bigint, + kept_at timestamptz not null default now() +); diff --git a/testdata/store-tests b/testdata/store-tests index d604f6d9..98b101b2 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -12,3 +12,5 @@ ./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore ./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain ./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance +./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket +./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds