The controller no longer subscribes to the alive message: HostBeats, its handler, the heartbeat-gap duration and a fixed silent bound (issue 1183, AC3, AC4, AC5)

cmd/mesh-controller/durations.go

@@ -72,8 +72,6 @@ func summarise(ds []inventory.Duration) []durationSummary {
 		switch k.kind {
 		case inventory.DurationApply:
 			s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
-		case inventory.DurationHeartbeatGap:
-			s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
 		}
 		out = append(out, s)
 	}
@@ -138,7 +136,7 @@ func durationsCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(summary) == 0 {
-		fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
+		fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
 			"plan-tier and build durations as it hears them\n", *days)
 		return nil
 	}

cmd/mesh-controller/handacts_test.go

@@ -77,15 +77,11 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
 		ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
 			Took: time.Duration(i) * time.Second})
 	}
-	ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
 	got := summarise(ds)
-	if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
+	if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
 		got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
 		t.Fatalf("%+v", got)
 	}
-	if !strings.Contains(got[1].Suggests, "3m0s") {
-		t.Fatalf("a minute between words suggests %q", got[1].Suggests)
-	}
 }
 
 // **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses

cmd/mesh-controller/last_heard_integration_test.go

@@ -0,0 +1,82 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// 1183/AC5: with the watchman's presence bucket holding a node online and no alive message ever sent, the node reads here
+// in `nodes` four minutes later by the controller's clock, however old its last report; and an alive message still sent
+// by a node-engine is published to the same bus without error and changes nothing.
+func TestAnOnlineNodeReadsHereWithNoAliveMessage(t *testing.T) { // 1183/AC5
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if _, err := broker.RaiseBuckets(js, []broker.Bucket{
+		{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	put := func(node string, online bool) {
+		t.Helper()
+		key, err := sdkdelivery.PresenceNodeKey(node)
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, err := sdkdelivery.EncodePresence(sdkdelivery.Presence{Online: online})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := kv.Put(ctx, key, body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	put("ace", true)
+	put("laptop", false)
+	// A node-engine still sending its alive message: harmless.
+	if err := js.Conn().Publish("mesh.control.ace.alive", []byte(`{"node":"ace"}`)); err != nil {
+		t.Fatalf("an alive message could not be published: %v", err)
+	}
+	if err := js.Conn().Flush(); err != nil {
+		t.Fatal(err)
+	}
+
+	later := time.Now().Add(4 * time.Minute)
+	nodes := []inventory.Node{{Name: "ace", LastSeen: time.Now().Add(-time.Hour)},
+		{Name: "laptop", LastSeen: time.Now().Add(-time.Hour)}}
+	read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
+		return readPresence(ctx, js.Conn(), names)
+	}
+	heard, err := nodesHeard(ctx, nodes, read, later)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := heardFromAt(heard[0], later); got != "here" {
+		t.Fatalf("a node online by presence reads %q four minutes later", got)
+	}
+	if got := heardFromAt(heard[1], later); got == "here" {
+		t.Fatalf("a node offline by presence reads %q", got)
+	}
+}

cmd/mesh-controller/last_heard_test.go

@@ -38,7 +38,7 @@ func TestLastHeardIsPresenceThenTheReport(t *testing.T) { // 1183/AC1
 // The machines facts, the nodes list, node and status all take that one function.
 func TestEveryReaderOfLastHeardTakesPresence(t *testing.T) { // 1183/AC1
 	now := time.Now()
-	m := machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute), every: time.Minute}
+	m := machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
 	applyPresence(&m, nil, &presenceEntry{At: now.Add(-4 * time.Minute)}, now)
 	if !m.lastHeard.Equal(now.Add(-4 * time.Minute)) {
 		t.Fatalf("machine facts take a report over the offline entry: %v", m.lastHeard)
@@ -78,7 +78,7 @@ func TestOnlineByPresenceReadsHereWhateverTheReport(t *testing.T) { // 1183/AC2
 	if len(quiet) != 1 || quiet[0].Name != "fresh" {
 		t.Errorf("quiet is %+v: an online machine is not quiet, and a machine offline 4 minutes is not quiet yet", quiet)
 	}
-	f := &machineFacts{name: "anchor", lastHeard: nodes[0].LastSeen, every: time.Minute}
+	f := &machineFacts{name: "anchor", lastHeard: nodes[0].LastSeen}
 	if f.silent(now) {
 		t.Errorf("a machine online by presence is silent")
 	}
@@ -95,3 +95,18 @@ func TestUnreadablePresenceLeavesTheReport(t *testing.T) { // 1183/AC1
 		t.Fatalf("the report was replaced: %v", nodes[0].LastSeen)
 	}
 }
+
+// 1183/AC4: where the watchman gives no verdict a machine is silent after a fixed three minutes, whatever interval a
+// node-engine once said.
+func TestTheSilentBoundIsFixed(t *testing.T) { // 1183/AC4
+	now := time.Now()
+	for _, c := range []struct {
+		ago  time.Duration
+		want bool
+	}{{179 * time.Second, false}, {181 * time.Second, true}} {
+		m := machineFacts{name: "laptop", lastHeard: now.Add(-c.ago)}
+		if got := m.silent(now); got != c.want {
+			t.Errorf("heard %v ago: silent %v, want %v", c.ago, got, c.want)
+		}
+	}
+}

cmd/mesh-controller/signals.go

@@ -318,6 +318,10 @@ func ago(d time.Duration) string {
 	return d.Round(time.Minute).String()
 }
 
+// silentBound is how long a machine may go unheard, where the watchman gives no verdict: three of the intervals nodes have
+// always used, fixed now that no machine says its own (issue 1183).
+const silentBound = heartbeatsMissed * heartbeatEvery
+
 // heartbeatBound is a machine's S1 or S11 bound from the interval it says.
 func heartbeatBound(every time.Duration) time.Duration {
 	if every <= 0 {
@@ -333,7 +337,7 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
 			// Never heard is a machine that has not joined, which status says; asleep is not lost.
 			continue
 		}
-		bound := heartbeatBound(m.every)
+		bound := silentBound
 		silent := f.now.Sub(m.lastHeard)
 		if !m.silent(f.now) {
 			continue

cmd/mesh-controller/signals_test.go

@@ -28,7 +28,7 @@ import (
 func calm(now time.Time) *signalFacts {
 	return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
 		machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
-			every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
+			sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
 			lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
 		plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
 			entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},

cmd/mesh-controller/stuck_probe.go

@@ -65,7 +65,7 @@ func (m machineFacts) silent(now time.Time) bool {
 	case presenceHeard:
 		return false
 	}
-	return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every)
+	return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > silentBound
 }
 
 // stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind

cmd/mesh-controller/stuck_probe_integration_test.go

@@ -72,7 +72,7 @@ func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { /
 	if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
 		t.Fatalf("a stuck key written as moved: %+v %v", got, err)
 	}
-	machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour), every: time.Minute}}
+	machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour)}}
 	if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-controller/stuck_probe_test.go

@@ -60,7 +60,7 @@ func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T)
 }
 
 func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
-	return machineFacts{name: "laptop", lastHeard: now.Add(-ago), every: time.Minute}
+	return machineFacts{name: "laptop", lastHeard: now.Add(-ago)}
 }
 
 // 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
@@ -97,7 +97,7 @@ func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
 func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
 	now := time.Now()
 	seen := now.Add(-time.Hour)
-	m := machineFacts{name: "laptop", lastHeard: seen, every: time.Minute}
+	m := machineFacts{name: "laptop", lastHeard: seen}
 	applyPresence(&m, nil, nil, now)
 	if !m.lastHeard.Equal(seen) {
 		t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
@@ -107,13 +107,13 @@ func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
 	if !m.lastHeard.Equal(offlineAt) {
 		t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
 	}
-	m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute), every: time.Minute}
+	m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
 	applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
 	if !m.lastHeard.Equal(now.Add(-time.Hour)) {
 		t.Fatalf("an offline entry's time is last heard, not a report: %v", m.lastHeard)
 	}
 	for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
-		m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen), every: time.Minute}
+		m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen)}
 		applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
 		if m.silent(now) {
 			t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)

cmd/mesh-controller/watchdogs.go

@@ -142,9 +142,7 @@ type machineFacts struct {
 	control bool
 	// lastHeard is the store's last word from it, any word; zero when never.
 	lastHeard time.Time
-	// every is the heartbeat interval it said; zero when it said none.
-	every time.Duration
-	power link.PowerState
+	power     link.PowerState
 	// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
 	sentAt          time.Time
 	reportedCurrent bool
@@ -457,9 +455,6 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
 	for _, n := range nodes {
 		m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
 			lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
-		if beat, ok := link.HostBeats.Of(n.Name); ok {
-			m.every = beat.Every
-		}
 		if beat, ok := link.ToolsBeats.Of(n.Name); ok {
 			m.toolsHeard, m.toolsEvery = beat.At, beat.Every
 		}