Merge pull request 'Keep a condition's looks in the store, so a look that changes nothing writes no bucket (issue 420)' (#249) from fix/420-conditions-bucket-on-transition into main

cmd/mesh-controller/condition_looks.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
+// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
+// is written when what a condition says changes, never for a look that saw it as it was.
+
+// storeLooks keeps the conditions' looks in the inventory.
+type storeLooks struct{ inv *inventory.Inventory }
+
+func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
+	evidence, err := json.Marshal(l.Evidence)
+	if err != nil {
+		return err
+	}
+	return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
+		LastObserved: l.LastObserved, Evidence: evidence})
+}
+
+func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
+	row, kept, err := s.inv.ConditionLooksOf(ctx, key)
+	if err != nil || !kept {
+		return conditions.Looks{}, kept, err
+	}
+	l, err := looksOf(row)
+	return l, err == nil, err
+}
+
+func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
+	rows, err := s.inv.AllConditionLooks(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]conditions.Looks, len(rows))
+	for key, row := range rows {
+		l, err := looksOf(row)
+		if err != nil {
+			return nil, err
+		}
+		out[key] = l
+	}
+	return out, nil
+}
+
+func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
+	return s.inv.ForgetConditionLooks(ctx, key)
+}
+
+func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
+	l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
+	if len(row.Evidence) > 0 {
+		if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
+			return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
+		}
+	}
+	return l, nil
+}
+
+// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
+// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
+// read through it, never read as none.
+func looksHere(ctx context.Context) (conditions.LookStore, func()) {
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return conditions.NoLooks{Err: err}, func() {}
+	}
+	return storeLooks{inv}, inv.Close
+}

cmd/mesh-controller/condition_looks_test.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
+// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
+// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
+// times in two days.
+func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
+	inv := inventory.ForTest(t)
+	bus := testbus.Start(t)
+	serving, err := broker.Dial(bus.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer serving.Close()
+	if err := serving.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	ctx := t.Context()
+	store, history, err := conditions.OnTheBus(ctx, serving.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
+	defer k.Close(context.Background())
+	look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
+		Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
+	for range 4 {
+		if _, err := k.Observe(ctx, look); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	api, err := jetstream.New(serving.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, err := stream.Info(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.State.LastSeq != 1 {
+		t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
+	}
+
+	// The verb as a process other than the serving controller: it reads the bucket and the store.
+	keptBefore := conditionsFrom
+	conditionsFrom = nil
+	defer func() { conditionsFrom = keptBefore }()
+	t.Setenv(broker.NATSVar, bus.ClientURL())
+	t.Setenv(broker.CertificateVar, "")
+	var shown bytes.Buffer
+	if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
+		t.Fatal(err)
+	}
+	var c conditions.Condition
+	if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
+		t.Fatalf("%v: %s", err, shown.String())
+	}
+	if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
+		time.Since(c.LastObserved) > time.Minute {
+		t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
+			len(c.Evidence), c.LastObserved, c.LooksUnread)
+	}
+	listed, err := openConditions(ctx)
+	if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
+		t.Fatalf("conditions lists %+v: %v", listed, err)
+	}
+}

cmd/mesh-controller/conditions.go

@@ -24,16 +24,18 @@ import (
 // **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
 // core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
 // by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
-// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
-// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
+// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
+// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
+// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
 // observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
 // while, with a reason, and that is recorded as a hand act.
 
 // conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
 var conditionsFrom *conditions.Keeper
 
-// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
-func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
+// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
+// the conditions' looks kept in looks (condition_looks.go).
+func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
 	store, history, err := conditions.OnTheBus(ctx, conn)
 	if err != nil {
 		return nil, err
@@ -42,7 +44,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error)
 	if err != nil {
 		return nil, err
 	}
-	return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
+	return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
 		Teller: link.OverNATS{Conn: conn, JS: js},
 		Say:    func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
 		// What status leads with changed: composed again soon (a nudge outside the serving controller
@@ -58,8 +60,10 @@ func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
 	if conditionsFrom != nil {
 		return f(conditionsFrom)
 	}
+	looks, closeLooks := looksHere(ctx)
+	defer closeLooks()
 	return onTheBus(func(conn *nats.Conn) error {
-		k, err := keeperOn(ctx, conn)
+		k, err := keeperOn(ctx, conn, looks)
 		if err != nil {
 			return err
 		}
@@ -82,6 +86,8 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
 		return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
 	}
 	var out []conditions.Condition
+	looks, closeLooks := looksHere(ctx)
+	defer closeLooks()
 	err := onTheBus(func(conn *nats.Conn) error {
 		store, _, err := conditions.OnTheBus(ctx, conn)
 		if err != nil {
@@ -89,7 +95,7 @@ func openConditions(ctx context.Context) ([]conditions.Condition, error) {
 		}
 		reading, cancel := context.WithTimeout(ctx, 5*time.Second)
 		defer cancel()
-		out, err = conditions.Read(reading, store)
+		out, err = conditions.ReadWith(reading, store, looks)
 		return err
 	})
 	return out, err
@@ -252,12 +258,14 @@ func showCondition(ctx context.Context, args []string, w io.Writer) error {
 	if conditionsFrom != nil {
 		c, found, err = conditionsFrom.Get(ctx, key)
 	} else {
+		looks, closeLooks := looksHere(ctx)
+		defer closeLooks()
 		err = onTheBus(func(conn *nats.Conn) error {
 			store, _, err := conditions.OnTheBus(ctx, conn)
 			if err != nil {
 				return err
 			}
-			c, found, err = conditions.ReadOne(ctx, store, key)
+			c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
 			return err
 		})
 	}

cmd/mesh-controller/doctor.go

@@ -651,7 +651,7 @@ func localDoctor(ctx context.Context) (*doctor, func(), error) {
 		open.Close()
 		return nil, nil, err
 	}
-	k, err := keeperOn(ctx, js.Conn())
+	k, err := keeperOn(ctx, js.Conn(), storeLooks{open.inventory})
 	if err != nil {
 		js.Close()
 		open.Close()

cmd/mesh-controller/replay327_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nats-io/nats-server/v2/server"
 
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/testbus"
 )
 
@@ -30,7 +31,7 @@ func TestReplay327(t *testing.T) {
 	if err := serving.EnsureControllerBuckets(); err != nil {
 		t.Fatal(err)
 	}
-	keeper, err := keeperOn(context.Background(), serving.Conn())
+	keeper, err := keeperOn(context.Background(), serving.Conn(), conditions.NewMemoryLooks())
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-controller/watchdogs.go

@@ -704,7 +704,7 @@ func later(a, b time.Time) time.Time {
 // self-check, for the serving controller; the returned function stops them. Nil when the store could
 // not be opened, which is said.
 func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
-	keeper, err := keeperOn(ctx, bus.Conn)
+	keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory})
 	if err != nil {
 		fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
 			"status says the conditions cannot be read: %v\n", err)