Keep a condition's looks in the store, so a look that changes nothing writes no bucket (issue 420)

Every look rewrote the condition's value in the conditions bucket, because the
value carried the count of looks, the last look and the newest evidence: 131 647
writes in four days for twelve open conditions, against ADR 0289's rule that a
bucket is written only when what it describes changed. The looks now live in the
controller's store (migration 0095, condition_looks), the bucket holds what a
condition says and is written only when that changes, and the conditions verb and
the condition events put the looks back.

cmd/mesh-controller/condition_looks.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
+// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
+// is written when what a condition says changes, never for a look that saw it as it was.
+
+// storeLooks keeps the conditions' looks in the inventory.
+type storeLooks struct{ inv *inventory.Inventory }
+
+func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
+	evidence, err := json.Marshal(l.Evidence)
+	if err != nil {
+		return err
+	}
+	return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
+		LastObserved: l.LastObserved, Evidence: evidence})
+}
+
+func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
+	row, kept, err := s.inv.ConditionLooksOf(ctx, key)
+	if err != nil || !kept {
+		return conditions.Looks{}, kept, err
+	}
+	l, err := looksOf(row)
+	return l, err == nil, err
+}
+
+func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
+	rows, err := s.inv.AllConditionLooks(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]conditions.Looks, len(rows))
+	for key, row := range rows {
+		l, err := looksOf(row)
+		if err != nil {
+			return nil, err
+		}
+		out[key] = l
+	}
+	return out, nil
+}
+
+func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
+	return s.inv.ForgetConditionLooks(ctx, key)
+}
+
+func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
+	l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
+	if len(row.Evidence) > 0 {
+		if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
+			return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
+		}
+	}
+	return l, nil
+}
+
+// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
+// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
+// read through it, never read as none.
+func looksHere(ctx context.Context) (conditions.LookStore, func()) {
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return conditions.NoLooks{Err: err}, func() {}
+	}
+	return storeLooks{inv}, inv.Close
+}

cmd/mesh-controller/condition_looks_test.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
+// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
+// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
+// times in two days.
+func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
+	inv := inventory.ForTest(t)
+	bus := testbus.Start(t)
+	serving, err := broker.Dial(bus.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer serving.Close()
+	if err := serving.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	ctx := t.Context()
+	store, history, err := conditions.OnTheBus(ctx, serving.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
+	defer k.Close(context.Background())
+	look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
+		Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
+	for range 4 {
+		if _, err := k.Observe(ctx, look); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	api, err := jetstream.New(serving.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, err := stream.Info(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.State.LastSeq != 1 {
+		t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
+	}
+
+	// The verb as a process other than the serving controller: it reads the bucket and the store.
+	keptBefore := conditionsFrom
+	conditionsFrom = nil
+	defer func() { conditionsFrom = keptBefore }()
+	t.Setenv(broker.NATSVar, bus.ClientURL())
+	t.Setenv(broker.CertificateVar, "")
+	var shown bytes.Buffer
+	if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
+		t.Fatal(err)
+	}
+	var c conditions.Condition
+	if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
+		t.Fatalf("%v: %s", err, shown.String())
+	}
+	if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
+		time.Since(c.LastObserved) > time.Minute {
+		t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
+			len(c.Evidence), c.LastObserved, c.LooksUnread)
+	}
+	listed, err := openConditions(ctx)
+	if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
+		t.Fatalf("conditions lists %+v: %v", listed, err)
+	}
+}

cmd/mesh-controller/conditions.go

@@ -24,16 +24,18 @@ import (
 // **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
 // core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
 // by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
-// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
-// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
+// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
+// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
+// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
 // observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
 // while, with a reason, and that is recorded as a hand act.
 
 // conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
 var conditionsFrom *conditions.Keeper
 
-// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
-func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
+// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
+// the conditions' looks kept in looks (condition_looks.go).
+func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
 	store, history, err := conditions.OnTheBus(ctx, conn)
 	if err != nil {
 		return nil, err
@@ -42,7 +44,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error)
 	if err != nil {
 		return nil, err
 	}
-	return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
+	return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
 		Teller: link.OverNATS{Conn: conn, JS: js},
 		Say:    func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
 		// What status leads with changed: composed again soon (a nudge outside the serving controller
@@ -58,8 +60,10 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
 	if conditionsFrom != nil {
 		return f(conditionsFrom)
 	}
+	looks, closeLooks := looksHere(ctx)
+	defer closeLooks()
 	return onTheBus(func(conn *nats.Conn) error {
-		k, err := keeperOn(ctx, conn)
+		k, err := keeperOn(ctx, conn, looks)
 		if err != nil {
 			return err
 		}
@@ -82,6 +86,8 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
 		return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
 	}
 	var out []conditions.Condition
+	looks, closeLooks := looksHere(ctx)
+	defer closeLooks()
 	err := onTheBus(func(conn *nats.Conn) error {
 		store, _, err := conditions.OnTheBus(ctx, conn)
 		if err != nil {
@@ -89,7 +95,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
 		}
 		reading, cancel := context.WithTimeout(ctx, 5*time.Second)
 		defer cancel()
-		out, err = conditions.Read(reading, store)
+		out, err = conditions.ReadWith(reading, store, looks)
 		return err
 	})
 	return out, err
@@ -252,12 +258,14 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error {
 	if conditionsFrom != nil {
 		c, found, err = conditionsFrom.Get(ctx, key)
 	} else {
+		looks, closeLooks := looksHere(ctx)
+		defer closeLooks()
 		err = onTheBus(func(conn *nats.Conn) error {
 			store, _, err := conditions.OnTheBus(ctx, conn)
 			if err != nil {
 				return err
 			}
-			c, found, err = conditions.ReadOne(ctx, store, key)
+			c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
 			return err
 		})
 	}

cmd/mesh-controller/doctor.go

@@ -651,7 +651,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) {
 		open.Close()
 		return nil, nil, err
 	}
-	k, err := keeperOn(ctx, js.Conn())
+	k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory})
 	if err != nil {
 		js.Close()
 		open.Close()

cmd/mesh-controller/replay327_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nats-io/nats-server/v2/server"
 
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/testbus"
 )
 
@@ -30,7 +31,7 @@ func TestReplay327(t *testing.T) {
 	if err := serving.EnsureControllerBuckets(); err != nil {
 		t.Fatal(err)
 	}
-	keeper, err := keeperOn(context.Background(), serving.Conn())
+	keeper, err := keeperOn(context.Background(), serving.Conn(), conditions.NewMemoryLooks())
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-controller/watchdogs.go

@@ -704,7 +704,7 @@ func later(a, b time.Time) time.Time {
 // self-check, for the serving controller; the returned function stops them. Nil when the store could
 // not be opened, which is said.
 func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
-	keeper, err := keeperOn(ctx, bus.Conn)
+	keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory})
 	if err != nil {
 		fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
 			"status says the conditions cannot be read: %v\n", err)