novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Give the controller lease back at the stop signal and say a takeover once, so a controller delivery raises no urgent alarm (issue 906, ADR 0558)

Browse Source 
The 2026-10-26 replacements showed the lease was given back, but last: the
shutdown took 10 s, the gate had closed at half the key's age, and the release
recorded the epoch released only through the gate, so the next holder found
it expired and S12 said it, urgent, for an hour.

Now the stop signal gives the key back first and records the epoch released
whenever the delete succeeded; a candidate reads the key's delete marker on
the bus to tell a handover from an expiry; and a real takeover (lost or
expired) is raised and cleared once by the controller that took it, instead
of an hour of S12 observation.

Research 072 piece moved: the lease's takeover read from its own key on the
bus and said as an event (03-domains, mesh-controller row; 04-findings 2,
the watchdogs' age reading).
Quick fix weighed: drop the gate check from the release record only; it
leaves the release last in the shutdown and the hour-long alarm for a real
takeover.
This commit is contained in:
jschoubben
2026-10-27 00:49:13 +01:00
parent 2dafbe60e9
commit fbaa463a38
8 changed files with 262 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/signals.go
+103 -15
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
@@ -147,9 +148,10 @@ var signalsTable = []signalRow{
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
}},
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
"the lease, for as long as it does",
Bound: "15 s (the key's age); a holder that lost the lease or stopped renewing it is said once by the " +
"controller that took it over, raised and cleared as it takes it, and for an hour after while none " +
"holds it; a lease given back is never said; a lease bucket found raised again from nothing is said " +
"for an hour after; a controller serving without the lease, for as long as it does",
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
@@ -644,10 +646,29 @@ func watchLease(f *signalFacts) []conditions.Observation {
"does not know it lost it",
Said: l.resetSaid})
}
// An epoch below the one this controller holds was taken over by it: said once by the takeover itself,
// raised and cleared (sayTakeover), not observed for an hour after (novox/hq issue 906). One that ended
// while this controller does not hold the lease — serving without it — is still said for the hour.
var ended []inventory.Epoch
for _, e := range l.ended {
if l.held && e.Epoch < l.epoch {
continue
}
ended = append(ended, e)
}
if o, ok := lostObservation(ended, l.epoch, f.now); ok {
out = append(out, o)
}
return out
}
// lostObservation is the lease-lost condition of the epochs that lost the lease or let it expire within the
// hour, named by the newest; false when none did. An epoch given back is no loss.
func lostObservation(ended []inventory.Epoch, acting uint64, now time.Time) (conditions.Observation, bool) {
var lost []string
newest := inventory.Epoch{}
for _, e := range l.ended {
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
for _, e := range ended {
if e.Ended == nil || e.How == inventory.EpochReleased || now.Sub(*e.Ended) > advisoryQuiet {
continue
}
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
@@ -656,18 +677,85 @@ func watchLease(f *signalFacts) []conditions.Observation {
newest = e
}
}
if len(lost) > 0 {
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
if newest.How == inventory.EpochExpired {
how = "stopped renewing it without giving it back, and was taken over"
if len(lost) == 0 {
return conditions.Observation{}, false
}
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
if newest.How == inventory.EpochExpired {
how = "stopped renewing it without giving it back, and was taken over"
}
return conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent, Source: "S12",
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
newest.Instance, how, acting),
Said: strings.Join(lost, "; ")}, true
}
// takeover is the condition a takeover says: the epoch just before the one taken, when it lost the lease or
// let it expire; false when it gave it back, or none is recorded.
func takeover(ended []inventory.Epoch, taken uint64, now time.Time) (conditions.Observation, bool) {
var before *inventory.Epoch
for i, e := range ended {
if e.Epoch < taken && (before == nil || e.Epoch > before.Epoch) {
before = &ended[i]
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
newest.Instance, how, l.epoch),
Said: strings.Join(lost, "; ")})
}
if before == nil {
return conditions.Observation{}, false
}
return lostObservation([]inventory.Epoch{*before}, taken, now)
}
// sayTakeover is the takeover said as an event (novox/hq issue 906): when the controller before this one lost
// the lease or let it expire, the lease-lost condition is raised and cleared at once by this controller, which
// holds the lease — the operator reads that it happened and that it is over, not an urgent alarm for an hour.
// A lease given back says nothing. Said once the condition store is open; a take before that is said then.
func (a *actor) sayTakeover(ctx context.Context, epoch uint64) {
a.mu.Lock()
keeper, inv := a.keeper, a.inv
a.mu.Unlock()
if keeper == nil || inv == nil || epoch == 0 {
return
}
now := time.Now()
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
ended, err := inv.EpochsSince(reading, now.Add(-advisoryQuiet))
if err != nil {
fmt.Printf("how the epoch before %d ended cannot be read, so its takeover is not said: %v\n", epoch, err)
return
}
sayTakeoverOf(reading, keeper, ended, epoch, now)
}
// sayTakeoverOf raises and clears the takeover's condition over the epochs read.
func sayTakeoverOf(ctx context.Context, keeper *conditions.Keeper, ended []inventory.Epoch, epoch uint64, now time.Time) {
o, ok := takeover(ended, epoch, now)
if !ok {
return
}
if _, err := keeper.Observe(ctx, o); err != nil {
fmt.Printf("the takeover at epoch %d could not be said: %v\n", epoch, err)
return
}
if _, err := keeper.Clear(ctx, o.Key(), fmt.Sprintf("taken over: the controller of epoch %d holds the lease", epoch)); err != nil {
fmt.Printf("the takeover at epoch %d was said and could not be cleared: %v\n", epoch, err)
}
}
// watchTakeovers gives the serving controller's condition store to the lease, and says the takeover of the
// epoch it holds, if it took one over before the store opened.
func (a *actor) watchTakeovers(ctx context.Context, inv *inventory.Inventory, keeper *conditions.Keeper) {
a.mu.Lock()
a.keeper, a.inv = keeper, inv
held := a.held
a.mu.Unlock()
if held == nil {
return
}
if epoch, err := held.Epoch(); err == nil {
a.sayTakeover(ctx, epoch)
}
return out
}
// handActsWithin is how far back a repeated cause counts (S15).